Further refactoring of algolia indexing

packages/website/mdx/algolia_helpers.ts

@@ -7,7 +7,7 @@ const filter = require('unist-util-filter');
 const { selectAll } = require('unist-util-select');
 const extractMdxMeta = require('extract-mdx-metadata');
 
-function processContentTree(tree: Node[], url: string, meta: Meta, index: any): void {
+function processContentTree(tree: Node[], url: string, meta: Meta, index: any, settings: Settings): void {
     const filteredTree = filter(tree, () => {
         return (node: Node) => node.type === 'heading' || node.type === 'paragraph';
     });
@@ -18,11 +18,12 @@ function processContentTree(tree: Node[], url: string, meta: Meta, index: any):
         const formattedTextNodes = formatTextNodes(textNodes);
         const content = getContent(meta, url, formattedTextNodes);
 
+        setIndexSettings(index, settings);
         pushObjectsToAlgolia(index, content);
     }
 }
 
-export function setIndexSettings(index: any, settings: Settings): void {
+function setIndexSettings(index: any, settings: Settings): void {
     index.setSettings(settings, (err: string) => {
         if (err) {
             throw Error(`Error: ${err}`);
@@ -80,7 +81,7 @@ function formatTextNodes(textNodes: Node[]): FormattedNode[] {
     return formattedTextNodes;
 }
 
-async function processMdxAsync(index: any, dirName: string, fileName: string): Promise<void> {
+async function processMdxAsync(index: any, dirName: string, fileName: string, settings: Settings): Promise<void> {
     const filePath = `${dirName}/${fileName}`;
     const { name } = path.parse(filePath); // Name without file extension
     const url = `/docs/${dirName}/${name}`;
@@ -91,14 +92,14 @@ async function processMdxAsync(index: any, dirName: string, fileName: string): P
 
     await remark()
         .use(mdx)
-        .use(() => (tree: Node[]) => processContentTree(tree, url, meta, index))
+        .use(() => (tree: Node[]) => processContentTree(tree, url, meta, index, settings))
         .process(file);
 }
 
-export async function indexFilesAsync(index: any, dirName: string): Promise<void> {
+export async function indexFilesAsync(index: any, dirName: string, settings: Settings): Promise<void> {
     fs.readdir(dirName, async (err: string, items: string[]) => {
         for (const fileName of items) {
-            await processMdxAsync(index, dirName, fileName);
+            await processMdxAsync(index, dirName, fileName, settings);
         }
     });
 }

packages/website/mdx/algolia_index.ts

@@ -0,0 +1,21 @@
+import { adminClient, searchIndices, settings } from '../ts/utils/algolia_search';
+
+import { indexFilesAsync } from './algolia_helpers';
+
+// Get args after command (i.e. ts-node) and path to file (i.e. ./index.ts)
+const args = process.argv.slice(2);
+
+function processIndices(indices: string[]): void {
+    for (const indexName of indices) {
+        const index = adminClient.initIndex(searchIndices[indexName]);
+        indexFilesAsync(index, indexName, settings[indexName]);
+    }
+}
+
+if (args.length > 0) {
+    // Use args given to process and push to algolia
+    processIndices(args);
+} else {
+    // Process and push all indices
+    processIndices(Object.getOwnPropertyNames(searchIndices));
+}

packages/website/mdx/guides.ts

@@ -1,8 +0,0 @@
-import { adminClient, searchIndex, settings } from '../ts/utils/algolia_search';
-import { indexFilesAsync, setIndexSettings } from './helpers';
-
-const indexName = 'guides';
-const index = adminClient.initIndex(searchIndex[indexName]);
-
-setIndexSettings(index, settings[indexName]);
-indexFilesAsync(index, indexName);

packages/website/mdx/tools.js

@@ -1,8 +0,0 @@
-import { adminClient, searchIndex, settings } from '../ts/utils/algolia_search';
-import { indexFilesAsync, setIndexSettings } from './helpers';
-
-const indexName = 'tools';
-const index = adminClient.initIndex(searchIndex[indexName]);
-
-setIndexSettings(index, settings[indexName]);
-indexFilesAsync(index, indexName);

packages/website/ts/components/docs/search/autocomplete.tsx

@@ -7,7 +7,7 @@ import { Link } from '@0x/react-shared';
 import { AutocompleteOverlay } from 'ts/components/docs/search/autocomplete_overlay';
 import { AutocompleteWrapper } from 'ts/components/docs/search/autocomplete_wrapper';
 
-import { searchIndex } from 'ts/utils/algolia_search';
+import { searchIndices } from 'ts/utils/algolia_search';
 
 interface IHit {
     description: string;
@@ -87,7 +87,7 @@ const CustomAutoComplete: React.FC<IAutoCompleteProps> = ({ isHome = false, hits
     };
 
     const renderSectionTitle = (section: any): React.ReactNode => {
-        const { tools, guides } = searchIndex;
+        const { tools, guides } = searchIndices;
 
         const titles: { [key: string]: string } = {
             [tools]: 'Tools',

packages/website/ts/components/docs/search/search_input.tsx

@@ -3,17 +3,17 @@ import { Configure, Index, InstantSearch } from 'react-instantsearch-dom';
 
 import { AutoComplete } from 'ts/components/docs/search/autocomplete';
 
-import { searchClient, searchIndex } from 'ts/utils/algolia_search';
+import { searchClient, searchIndices } from 'ts/utils/algolia_search';
 
 interface ISearchInputProps {
     isHome?: boolean;
 }
 
 export const SearchInput: React.FC<ISearchInputProps> = ({ isHome }) => (
-    <InstantSearch searchClient={searchClient} indexName={searchIndex.tools}>
+    <InstantSearch searchClient={searchClient} indexName={searchIndices.tools}>
         <AutoComplete isHome={isHome} />
         <Configure hitsPerPage={5} distinct={true} />
-        <Index indexName={searchIndex.tools} />
-        <Index indexName={searchIndex.guides} />
+        <Index indexName={searchIndices.tools} />
+        <Index indexName={searchIndices.guides} />
     </InstantSearch>
 );

packages/website/ts/pages/docs/guides.tsx

@@ -7,11 +7,11 @@ import { Resource } from 'ts/components/docs/resource/resource';
 import { Separator } from 'ts/components/docs/separator';
 import { Filters } from 'ts/components/docs/sidebar/filters';
 
-import { searchClient, searchIndex } from 'ts/utils/algolia_search';
+import { searchClient, searchIndices } from 'ts/utils/algolia_search';
 
 export const DocsGuides: React.FC = () => (
     <DocsPageLayout title="Guides">
-        <InstantSearch searchClient={searchClient} indexName={searchIndex.guides}>
+        <InstantSearch searchClient={searchClient} indexName={searchIndices.guides}>
             <Columns>
                 <Filters filters={filters} />
                 <Separator />

packages/website/ts/pages/docs/tools.tsx

@@ -13,7 +13,7 @@ import { ContentWrapper } from 'ts/components/docs/layout/content_wrapper';
 import { DocsPageLayout } from 'ts/components/docs/layout/docs_page_layout';
 import { Separator } from 'ts/components/docs/separator';
 
-import { searchClient, searchIndex } from 'ts/utils/algolia_search';
+import { searchClient, searchIndices } from 'ts/utils/algolia_search';
 
 interface IHitsProps {
     hits: IHit[];
@@ -37,7 +37,7 @@ interface IHit {
 export const DocsTools: React.FC = () => {
     return (
         <DocsPageLayout title="Tools">
-            <InstantSearch searchClient={searchClient} indexName={searchIndex.tools}>
+            <InstantSearch searchClient={searchClient} indexName={searchIndices.tools}>
                 <Columns>
                     <Filters filters={filters} />
                     <Separator />

packages/website/ts/utils/algolia_search.ts

@@ -5,10 +5,27 @@ const ALGOLIA_CLIENT_API_KEY = '4c367b8cc6d6e175ae537cc61e4d8dfd';
 // @TODO: Move the following somewhere safe / out of the repo
 const ALGOLIA_ADMIN_API_KEY = 'ccc472dee2aa991ca4bc935975e76b5d';
 
+interface ISearchIndices {
+    [index: string]: string;
+}
+
+interface ISettingsIndex {
+    [index: string]: Settings;
+}
+
+interface Settings {
+    distinct: boolean;
+    attributeForDistinct: string;
+    attributesForFaceting: string[];
+    attributesToSnippet: string[];
+    searchableAttributes: string[];
+    snippetEllipsisText: string;
+}
+
 export const searchClient = algoliasearch(ALGOLIA_APP_ID, ALGOLIA_CLIENT_API_KEY);
 export const adminClient = algoliasearch(ALGOLIA_APP_ID, ALGOLIA_ADMIN_API_KEY);
 
-export const searchIndex = {
+export const searchIndices: ISearchIndices = {
     guides: '0x_guides_test',
     tools: '0x_tools_test',
 };
@@ -21,7 +38,7 @@ const sharedSettings = {
     snippetEllipsisText: '…',
 };
 
-export const settings = {
+export const settings: ISettingsIndex = {
     guides: {
         ...sharedSettings,
         attributesForFaceting: ['topics', 'difficulty'],