102 lines
3.1 KiB
JavaScript
102 lines
3.1 KiB
JavaScript
const { chain, chunk, difference } = require('lodash')
|
|
const eventToPromise = require('event-to-promise')
|
|
const objectHash = require('object-hash')
|
|
const rank = require('./rank')
|
|
const validateRecords = require('./validate-records')
|
|
const getAlgoliaClient = require('./algolia-client')
|
|
|
|
class AlgoliaIndex {
|
|
constructor (name, records) {
|
|
this.name = name
|
|
this.records = records
|
|
.map(record => {
|
|
record.customRanking = rank(record)
|
|
return record
|
|
})
|
|
this.validate()
|
|
return this
|
|
}
|
|
|
|
validate () {
|
|
return validateRecords(this.name, this.records)
|
|
}
|
|
|
|
// This method consumes Algolia's `browseAll` event emitter,
|
|
// aggregating results into an array of all the records
|
|
// https://www.algolia.com/doc/api-reference/api-methods/browse/
|
|
async fetchExistingRecords () {
|
|
const client = getAlgoliaClient()
|
|
|
|
// return an empty array if the index does not exist yet
|
|
const { items: indices } = await client.listIndices()
|
|
|
|
if (!indices.find(index => index.name === this.name)) {
|
|
console.log(`index '${this.name}' does not exist!`)
|
|
return []
|
|
}
|
|
|
|
const index = client.initIndex(this.name)
|
|
const browser = index.browseAll()
|
|
let records = []
|
|
|
|
browser.on('result', (content) => {
|
|
records = records.concat(content.hits)
|
|
})
|
|
|
|
browser.on('error', (err) => {
|
|
throw err
|
|
})
|
|
|
|
await eventToPromise(browser, 'end')
|
|
|
|
return records
|
|
}
|
|
|
|
async syncWithRemote () {
|
|
const client = getAlgoliaClient()
|
|
|
|
console.log('\n\nsyncing %s with remote', this.name)
|
|
this.validate()
|
|
|
|
const existingRecords = await this.fetchExistingRecords()
|
|
|
|
const existingIds = chain(existingRecords).map('objectID').value()
|
|
const currentIds = chain(this.records).map('objectID').value()
|
|
const deprecatedIds = difference(existingIds, currentIds)
|
|
const newIds = difference(currentIds, existingIds)
|
|
|
|
// Create a hash of every existing record, to compare to the new records
|
|
// The `object-hash` module is indifferent to object key order by default. :+1:
|
|
const existingHashes = existingRecords.map(record => objectHash(record))
|
|
|
|
// If a hash is found, that means the existing Algolia record contains the
|
|
// same data as new record, and the record doesn't need to be updated.
|
|
const recordsToUpdate = this.records.filter(record => {
|
|
return !existingHashes.includes(objectHash(record))
|
|
})
|
|
|
|
console.log('deprecated objectIDs:', deprecatedIds)
|
|
console.log('new objectIDs:', newIds)
|
|
console.log('total current records:', this.records.length)
|
|
console.log('records to update:', recordsToUpdate.length)
|
|
|
|
const index = client.initIndex(this.name)
|
|
|
|
if (deprecatedIds.length) {
|
|
console.log('deleting %d deprecated record(s)', deprecatedIds.length)
|
|
await index.deleteObjects(deprecatedIds)
|
|
}
|
|
|
|
if (recordsToUpdate.length) {
|
|
console.log('uploading %d new or modified record(s)', recordsToUpdate.length)
|
|
const chunks = chunk(recordsToUpdate, 1000)
|
|
|
|
for (const batch of chunks) {
|
|
await index.addObjects(batch)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
module.exports = AlgoliaIndex
|