Page MenuHomePhabricator
Differential D4483 Diff 22680 docs/next/src/scripts/processData.ts

Changeset View

Standalone View

View Options

docs/next/src/scripts/processData.ts

import fg from 'fast-glob'; import fg from 'fast-glob';
import path from 'path'; import path from 'path';
import { promises as fs } from 'fs'; import {promises as fs} from 'fs';
import algoliasearch from 'algoliasearch'; import algoliasearch from 'algoliasearch';
import data from '../data/searchindex.json'; import data from '../data/searchindex.json';
// Folder paths // Folder paths
const DATA_PATH = path.join(__dirname, '../data'); const DATA_PATH = path.join(__dirname, '../data');
const MODULE_PATH = path.join(DATA_PATH, '_modules'); const MODULE_PATH = path.join(DATA_PATH, '_modules');
const EXAMPLES_PATH = path.join(__dirname, '../../../../examples'); const EXAMPLES_PATH = path.join(__dirname, '../../../../examples');
const EXAMPLES_FOLDER_PATH = path.join(__dirname, '../pages/examples'); const EXAMPLES_FOLDER_PATH = path.join(__dirname, '../pages/examples');
process.on('unhandledRejection', (error) => { process.on('unhandledRejection', (error) => {
console.error(error); // This prints error with stack included (as for normal errors) console.error(error); // This prints error with stack included (as for normal errors)
throw error; throw error;
}); });
async function preProcess() { async function preProcess() {
const glob = path.join(DATA_PATH, '/**/*.fjson'); const glob = path.join(DATA_PATH, '/**/*.fjson');
const entries = await fg([glob]); const entries = await fg([glob]);
for (const entry of entries) { for (const entry of entries) {
await fs.rename(entry, entry.replace('fjson', 'json')); await fs.rename(entry, entry.replace('fjson', 'json'));
} }
console.log('✅ Converted fjson files to json'); console.log('✅ Converted fjson files to json');
} }
async function rewriteRelativeLinks() { async function rewriteRelativeLinks() {
/* Generate list of all module files */
const glob = path.join(DATA_PATH, '/**/*.json'); const glob = path.join(DATA_PATH, '/**/*.json');
const excludeGlob = path.join(MODULE_PATH, '/**/*.json'); const moduleGlob = path.join(MODULE_PATH, '/**/*.json');
const entries = await fg([glob], { ignore: [excludeGlob] });
for (const entry of entries) { const docEntries = await fg([glob], {ignore: [moduleGlob]});
const rawData = (await fs.readFile(entry)).toString(); for (const entry of docEntries) {
const fileData = JSON.parse(rawData); applyTransform(entry, transformDocLink);
const body: string = fileData.body; }
if (body) {
let transformed = body const moduleEntries = await fg([moduleGlob]);
for (const entry of moduleEntries) {
applyTransform(entry, transformModuleLink);
}
console.log('✅ Re-wrote all relative links');
}
function transformDocLink(filename: string, fileContent: string) {
let transformed = fileContent
.replace(/href="\.\.\/.*?(\/#.*?)"/g, (match) => { .replace(/href="\.\.\/.*?(\/#.*?)"/g, (match) => {
return match.replace('/#', '#'); return match.replace('/#', '#');
}) })
.replace(/href="(\.\.\/)[^.]/g, (match, p1) => { .replace(/href="(\.\.\/)[^.]/g, (match, p1) => {
return match.replace(p1, ''); return match.replace(p1, '');
}); });
if (entry.includes('/libraries/')) { if (filename.includes('/libraries/')) {
transformed = body.replace(/href="\.\.\/\.\.\//g, 'href="../'); transformed = fileContent.replace(/href="\.\.\/\.\.\//g, 'href="../');
} }
fileData.body = transformed;
await fs.writeFile(entry, JSON.stringify(fileData), 'utf8'); return transformed;
} }
function transformModuleLink(filename: string, fileContent: string) {
return fileContent.replace(/href="[^"]*"/g, (match) => {
return match.replace(/sections\/api\/apidocs\//g, '_apidocs/').replace('/#', '#');
});
} }
console.log('✅ Re-wrote all relative links');
async function applyTransform(
entry: string,
transformFn: (filename: string, content: string) => string,
) {
const rawData = (await fs.readFile(entry)).toString();
const fileData = JSON.parse(rawData);
const body: string = fileData.body;
if (!body) {
return;
}
fileData.body = transformFn(entry, body);
await fs.writeFile(entry, JSON.stringify(fileData), 'utf8');
} }
async function createModuleIndex() { async function createModuleIndex() {
/* Generate list of all module files */ /* Generate list of all module files */
const glob = path.join(MODULE_PATH, '/**/*.json'); const glob = path.join(MODULE_PATH, '/**/*.json');
const rootGlob = path.join(MODULE_PATH, '/*.json'); const rootGlob = path.join(MODULE_PATH, '/*.json');
const entries = await fg([glob]); const entries = await fg([glob]);
const excludeEntries = await fg([rootGlob]); const excludeEntries = await fg([rootGlob]);
const relativeEntries = entries const relativeEntries = entries
.filter((i) => !excludeEntries.includes(i)) .filter((i) => !excludeEntries.includes(i))
.map((i) => path.relative(MODULE_PATH, i).replace('.json', '')); .map((i) => path.relative(MODULE_PATH, i).replace('.json', ''));
const index = { const index = {
docnames: relativeEntries, docnames: relativeEntries,
}; };
await fs.writeFile( await fs.writeFile(path.join(MODULE_PATH, 'searchindex.json'), JSON.stringify(index));
path.join(MODULE_PATH, 'searchindex.json'),
JSON.stringify(index),
);
console.log('✅ Generated list of all list and module files'); console.log('✅ Generated list of all list and module files');
} }
const dirs = async (dirPath: string) => { const dirs = async (dirPath: string) => {
let dirs: string[] = []; let dirs: string[] = [];
for (const file of await fs.readdir(dirPath)) { for (const file of await fs.readdir(dirPath)) {
if ((await fs.stat(path.join(dirPath, file))).isDirectory()) { if ((await fs.stat(path.join(dirPath, file))).isDirectory()) {
dirs = [...dirs, file]; dirs = [...dirs, file];
} }
} }
return dirs; return dirs;
}; };
async function createAlgoliaIndex() { async function createAlgoliaIndex() {
const { NEXT_ALGOLIA_APP_ID, NEXT_ALGOLIA_ADMIN_KEY } = process.env; const {NEXT_ALGOLIA_APP_ID, NEXT_ALGOLIA_ADMIN_KEY} = process.env;
if (!NEXT_ALGOLIA_APP_ID || !NEXT_ALGOLIA_ADMIN_KEY) { if (!NEXT_ALGOLIA_APP_ID || !NEXT_ALGOLIA_ADMIN_KEY) {
if (process.env.NODE_ENV === 'production') { if (process.env.NODE_ENV === 'production') {
console.error( console.error(
'\x1b[31m%s\x1b[0m', // cyan '\x1b[31m%s\x1b[0m', // cyan
'ERROR: environment variable NEXT_ALGOLIA_APP_ID or NEXT_ALGOLIA_ADMIN_KEY not set.\n' + 'ERROR: environment variable NEXT_ALGOLIA_APP_ID or NEXT_ALGOLIA_ADMIN_KEY not set.\n' +
'Please use NODE_ENV=development if you are not deploying the next doc.', 'Please use NODE_ENV=development if you are not deploying the next doc.',
); );
Show All 23 Linesasync function createAlgoliaIndex() {
for (const module in objects) { for (const module in objects) {
for (const object in objects[module]) { for (const object in objects[module]) {
const objProperties = objects[module][object]; const objProperties = objects[module][object];
const docNamesIndex: number = objProperties[0] as number; const docNamesIndex: number = objProperties[0] as number;
const typeIndex: number = objProperties[1] as number; const typeIndex: number = objProperties[1] as number;
const type = objectNames[typeIndex][1]; const type = objectNames[typeIndex][1];
const alias: string = objProperties[3] as string; const alias: string = objProperties[3] as string;
const relativePath = data.docnames[docNamesIndex].replace( const relativePath = data.docnames[docNamesIndex].replace('sections/api/_apidocs/', '');
'sections/api/apidocs/',
'',
);
const path = const path =
alias === '' alias === '' ? relativePath + `#${module}.${object}` : relativePath + '#' + alias;
? relativePath + `#${module}.${object}`
: relativePath + '#' + alias;
const record = { const record = {
objectID: `${module}.${object}.${type}.${alias}`, objectID: `${module}.${object}.${type}.${alias}`,
module, module,
object, object,
path, path,
type, type,
alias, alias,
}; };
records.push(record); records.push(record);
} }
} }
try { try {
await index.saveObjects(records, { autoGenerateObjectIDIfNotExist: true }); await index.saveObjects(records, {autoGenerateObjectIDIfNotExist: true});
console.log('✅ Updated Algolia index'); console.log('✅ Updated Algolia index');
} catch { } catch {
console.log('❌ Updated Algolia index'); console.log('❌ Updated Algolia index');
} }
} }
const steps = [ const steps = [preProcess, rewriteRelativeLinks, createModuleIndex, createAlgoliaIndex];
preProcess,
rewriteRelativeLinks,
createModuleIndex,
createAlgoliaIndex,
];
(async () => { (async () => {
for (const step of steps) { for (const step of steps) {
await step(); await step();
} }
})(); })();