tmp file management

framework/agility/agility.config.js

@@ -1,54 +1,74 @@
 const agilityContentSync = require("@agility/content-sync");
-const agilityFileSystem = require("@agility/content-sync/src/store-interface-filesystem");
+const nextFileSystem = require("./next.file.sync");
+
+const path = require("path")
+const fs = require('fs-extra');
 
 const agilityConfig = {
-  guid: process.env.AGILITY_GUID, //Set your guid here
-  fetchAPIKey: process.env.AGILITY_API_FETCH_KEY, //Set your fetch apikey here
-  previewAPIKey: process.env.AGILITY_API_PREVIEW_KEY, //set your preview apikey
-  languageCode: "en-us", //the language for your website in Agility CMS
-  channelName: "website", //the name of your channel in Agility CMS
-  securityKey: process.env.AGILITY_SECURITY_KEY, //the website security key used to validate and generate preview keys
+	guid: process.env.AGILITY_GUID, //Set your guid here
+	fetchAPIKey: process.env.AGILITY_API_FETCH_KEY, //Set your fetch apikey here
+	previewAPIKey: process.env.AGILITY_API_PREVIEW_KEY, //set your preview apikey
+	languageCode: "en-us", //the language for your website in Agility CMS
+	channelName: "website", //the name of your channel in Agility CMS
+	securityKey: process.env.AGILITY_SECURITY_KEY, //the website security key used to validate and generate preview keys
 };
 
-const getSyncClient = ({ isPreview, isDevelopmentMode }) => {
+const getSyncClient = ({ isPreview, isDevelopmentMode, isIncremental }) => {
 
 	const rootPath = process.cwd()
 
-  let cachePath = `${rootPath}/.next/cache/agility/${
-    isPreview ? "preview" : "live"
-  }`;
+	let cachePath = `${rootPath}/.next/cache/agility/${isPreview ? "preview" : "live" }`;
 
-//   if (!isDevelopmentMode) {
-//     //we are in prod and need to use a different directory that Vercel understands
-//     cachePath = `/tmp/agilitycache/${isPreview ? "preview" : "live"}`;
-//   }
+	//if we are in "incremental" mode, we need to use the tmp folder...
+	if (isIncremental) {
+		cachePath = `/tmp/agilitycache/${isPreview ? "preview" : "live"}`;
+	}
 
-  console.log(`Agility CMS => Content cache path is ${cachePath}`);
-  const apiKey = isPreview
-    ? agilityConfig.previewAPIKey
-    : agilityConfig.fetchAPIKey;
+	console.log(`AgilityCMS => Content cache path is ${cachePath}`);
+	const apiKey = isPreview
+		? agilityConfig.previewAPIKey
+		: agilityConfig.fetchAPIKey;
 
-  if (!agilityConfig.guid) {
-    console.log("Agility CMS => No GUID was provided.");
-    return null;
-  }
+	if (!agilityConfig.guid) {
+		console.log("AgilityCMS => No GUID was provided.");
+		return null;
+	}
 
-  return agilityContentSync.getSyncClient({
-    guid: agilityConfig.guid,
-    apiKey: apiKey,
-    isPreview: isPreview,
-    languages: [agilityConfig.languageCode],
-    channels: [agilityConfig.channelName],
-    store: {
-      interface: agilityFileSystem,
-      options: {
-        rootPath: cachePath,
-      },
-    },
-  });
+	return agilityContentSync.getSyncClient({
+		guid: agilityConfig.guid,
+		apiKey: apiKey,
+		isPreview: isPreview,
+		languages: [agilityConfig.languageCode],
+		channels: [agilityConfig.channelName],
+		store: {
+			interface: nextFileSystem,
+			options: {
+				rootPath: cachePath
+			},
+		},
+	});
 };
 
+
+const prepIncrementalMode = async () => {
+
+	let cachePath = `${rootPath}/.next/cache/agility/`;
+	const tempPath = `/tmp/agilitycache/`;
+
+	const buildFilePath = path.join(tempPath, "build.log")
+
+	//check for the build file in here...
+	if (!fs.existsSync(buildFilePath)) {
+		console.log(`Copying Agility Content files to temp folder: ${tempPath}`)
+		//copy everything across from cachePath
+		await fs.copy(cachePath, tempPath)
+	}
+}
+
+
+
 module.exports = {
-  agilityConfig,
-  getSyncClient,
+	agilityConfig,
+	getSyncClient,
+	prepIncrementalMode
 };

framework/agility/agility.node.ts

@@ -1,10 +1,11 @@
+import fs from "fs"
 import crypto from 'crypto'
 import { asyncForEach } from "./utils"
 
 import { ModuleWithInit } from "@agility/types"
 
 //Agility API stuff
-import { agilityConfig, getSyncClient } from './agility.config'
+import { agilityConfig, getSyncClient, prepIncrementalMode } from './agility.config'
 import GlobalFooter from 'components/agility-global/GlobalFooter'
 import GlobalHeader from 'components/agility-global/GlobalHeader'
 
@@ -41,35 +42,37 @@ const getAgilityPageProps = async ({ params, preview, locale }:any):Promise<Agil
 		})
 	}
 
+	//determine if we've already done a full build yet
+	const buildFilePath = `${process.cwd()}/.next/cache/agility/build.log`
+	const isBuildComplete = fs.existsSync(buildFilePath)
+
 	//TODO: use locale to determin LANGUAGECODE (pulled from config at this point...)
 
 	//determine if we are in preview mode
 	const isPreview:boolean = (preview || isDevelopmentMode);
 
-
-
 	const agilitySyncClient = getSyncClient({
 		isPreview: isPreview,
-		isDevelopmentMode
+		isDevelopmentMode,
+		isIncremental: isBuildComplete
 	});
 
-
-	//always sync to get latest
-
-
 	if (! agilitySyncClient) {
-		console.log("Agility CMS => Sync client could not be accessed.")
+		console.log("AgilityCMS => Sync client could not be accessed.")
 		return {
 			notFound: true
 		};
 	}
 
-	if (!isDevelopmentMode) {
-		console.log(`Agility CMS => Syncing ${isPreview ? "Preview" : "Live"} Mode`)
+	if (preview || isBuildComplete) {
+		//only do on-demand sync in next's preview mode or incremental build...
+		console.log(`AgilityCMS => Sync On-demand ${isPreview ? "Preview" : "Live"} Mode`)
+
+		await prepIncrementalMode()
 		await agilitySyncClient.runSync();
 	}
 
-	console.log(`Agility CMS => Getting page props for '${path}'...`);
+	console.log(`AgilityCMS => Getting page props for '${path}'...`);
 
 	//get sitemap
 	const sitemap = await agilitySyncClient.store.getSitemap({ channelName, languageCode });
@@ -135,7 +138,7 @@ const getAgilityPageProps = async ({ params, preview, locale }:any):Promise<Agil
 
 				//we have some additional data in the module we'll need, execute that method now, so it can be included in SSG
 				if (isDevelopmentMode) {
-					console.log(`Agility CMS => Fetching additional data via getCustomInitialProps for ${moduleItem.module}...`);
+					console.log(`AgilityCMS => Fetching additional data via getCustomInitialProps for ${moduleItem.module}...`);
 				}
 
 				const moduleData = await moduleInitializer({
@@ -183,28 +186,28 @@ const getAgilityPageProps = async ({ params, preview, locale }:any):Promise<Agil
 	}
 }
 
-const getAgilityPaths = async () => {
+const getAgilityPaths = async (preview:boolean|undefined) => {
 
-	console.log(`Agility CMS => Fetching sitemap for getAgilityPaths...`);
+	console.log(`AgilityCMS => Fetching sitemap for getAgilityPaths...`);
 
 	//determine if we are in preview mode
-	const isPreview = isDevelopmentMode;
+	const isPreview = isDevelopmentMode || preview;
+
+	//determine if we've already done a full build yet
+	const buildFilePath = `${process.cwd()}/.next/cache/agility/build.log`
+	const isBuildComplete = fs.existsSync(buildFilePath)
 
 	const agilitySyncClient = getSyncClient({
 		isPreview,
-		isDevelopmentMode
+		isDevelopmentMode,
+		isIncremental: isBuildComplete
 	});
 
-	//always sync to get latest
 
 	if (! agilitySyncClient) {
-		console.log("Agility CMS => Sync client could not be accessed.")
+		console.log("AgilityCMS => Sync client could not be accessed.")
 		return [];
 	}
-	if (!isDevelopmentMode) {
-		console.log(`Agility CMS => Syncing ${isPreview ? "Preview" : "Live"} Mode`)
-		await agilitySyncClient.runSync();
-	}
 
 	const sitemapFlat = await agilitySyncClient.store.getSitemap({
 		channelName,
@@ -214,7 +217,7 @@ const getAgilityPaths = async () => {
 
 
 	if (!sitemapFlat) {
-		console.warn("Agility CMS => No Site map found.  Make sure your environment variables are setup correctly.")
+		console.warn("AgilityCMS => No Site map found.  Make sure your environment variables are setup correctly.")
 		return []
 	}
 

framework/agility/agility.sync.js

@@ -1,8 +1,5 @@
-/*
-THIS FILE IS ONLY EXECUTED LOCALLY
-WHEN DOING A LOCAL SYNC ON DEMAND
-IN DEVELOPMENT MODE
-*/
+
+const fs = require('fs')
 
 require("dotenv").config({
 	path: `.env.local`,
@@ -12,22 +9,44 @@ const { getSyncClient } = require('./agility.config')
 
 
 const runSync = async () => {
+	setBuildLog(false)
 
 	const agilitySyncClient = getSyncClient({ isPreview: true, isDevelopmentMode: true })
 	if (! agilitySyncClient) {
-		console.log("Agility CMS => Sync client could not be accessed.")
+		console.log("AgilityCMS => Sync client could not be accessed.")
 		return;
 	}
 
 	await agilitySyncClient.runSync();
 }
 
-const syncAll = async () => {
+const setBuildLog = (builtYN) => {
+	//clear out a file saying WE HAVE SYNC'D
+	const rootPath = process.cwd()
+	const filePath = `${rootPath}/.next/cache/agility/build.log`
+	if (fs.existsSync(filePath)) {
+		fs.unlinkSync(filePath);
+	}
+
+	if (builtYN) {
+		//write out the build log so we know that we are up to date
+		fs.writeFileSync(filePath, "BUILT");
+	} else {
+		if (fs.existsSync(filePath)) {
+			fs.unlinkSync(filePath);
+		}
+	}
+}
+
+const preBuild = async () => {
+
+	//clear the build log
+	setBuildLog(false)
 
 	//sync preview mode
 	let agilitySyncClient = getSyncClient({ isPreview: true, isDevelopmentMode: false })
 	if (! agilitySyncClient) {
-		console.log("Agility CMS => Sync client could not be accessed.")
+		console.log("AgilityCMS => Sync client could not be accessed.")
 		return;
 	}
 
@@ -36,18 +55,27 @@ const syncAll = async () => {
 	//sync production mode
 	agilitySyncClient = getSyncClient({ isPreview: false, isDevelopmentMode: false })
 	if (! agilitySyncClient) {
-		console.log("Agility CMS => Sync client could not be accessed.")
+		console.log("AgilityCMS => Sync client could not be accessed.")
 		return;
 	}
 
 	await agilitySyncClient.runSync();
+
+
+}
+
+const postBuild = async() => {
+	//mark the build log as BUILT
+	setBuildLog(true)
 }
 
 const clearSync = async () => {
 
+	setBuildLog(false)
+
 	const agilitySyncClient = getSyncClient({ isPreview: true, isDevelopmentMode: true })
 	if (! agilitySyncClient) {
-		console.log("Agility CMS => Sync client could not be accessed.")
+		console.log("AgilityCMS => Sync client could not be accessed.")
 		return;
 	}
 	await agilitySyncClient.clearSync();
@@ -63,10 +91,13 @@ if (process.argv[2]) {
 		//run the sync
 		return runSync()
 
-	} else if (process.argv[2] === "sync-all") {
-		//sync both staging and live content
-		return syncAll()
+	} else if (process.argv[2] === "prebuild") {
+		//pre build actions
+		return preBuild()
 
+	} else if (process.argv[2] === "postbuild") {
+		//post build actions
+		return postBuild()
 	}
 }
 

framework/agility/next.file.sync.js

@@ -0,0 +1,153 @@
+const fs = require('fs')
+const os = require('os')
+const path = require('path')
+
+const { lockSync, unlockSync, checkSync, check }  = require("proper-lockfile")
+
+require("dotenv").config({
+	path: `.env.${process.env.NODE_ENV}`,
+})
+
+const sleep = (ms) => {
+	return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+
+const getFilePath = ({ options, itemType, languageCode, itemID }) => {
+
+	const fileName = `${itemID}.json`;
+	return path.join(options.rootPath, languageCode, itemType, fileName);
+}
+
+const saveItem = async ({ options, item, itemType, languageCode, itemID }) => {
+
+	let filePath = getFilePath({ options, itemType, languageCode, itemID });
+
+	let dirPath = path.dirname(filePath);
+
+
+	if (!fs.existsSync(dirPath)) {
+		fs.mkdirSync(dirPath, { recursive: true });
+	}
+
+	let json = JSON.stringify(item);
+	fs.writeFileSync(filePath, json);
+}
+
+const deleteItem = async ({ options, itemType, languageCode, itemID }) => {
+
+	let filePath = getFilePath({ options, itemType, languageCode, itemID });
+
+	if (fs.existsSync(filePath)) {
+		fs.unlinkSync(filePath);
+	}
+
+}
+
+const mergeItemToList = async ({ options, item, languageCode, itemID, referenceName, definitionName }) => {
+
+	let contentList = await getItem({ options, itemType: "list", languageCode, itemID: referenceName });
+
+	if (contentList == null) {
+		//initialize the list
+		contentList = [item];
+	} else {
+		//replace the item...
+		const cIndex = contentList.findIndex((ci) => {
+			return ci.contentID === itemID;
+		});
+
+		if (item.properties.state === 3) {
+			//*** deleted item (remove from the list) ***
+			if (cIndex >= 0) {
+				//remove the item
+				contentList.splice(cIndex, 1);
+			}
+
+		} else {
+			//*** regular item (merge) ***
+			if (cIndex >= 0) {
+				//replace the existing item
+				contentList[cIndex] = item;
+			} else {
+				//and it to the end of the
+				contentList.push(item);
+			}
+		}
+	}
+
+	await saveItem({ options, item: contentList, itemType: "list", languageCode, itemID: referenceName });
+}
+
+const getItem = async ({ options, itemType, languageCode, itemID }) => {
+	let filePath = getFilePath({ options, itemType, languageCode, itemID });
+
+	if (!fs.existsSync(filePath)) return null;
+
+	let json = fs.readFileSync(filePath, 'utf8');
+	return JSON.parse(json);
+}
+
+const clearItems = async ({ options }) => {
+	fs.rmdirSync(options.rootPath, { recursive: true })
+}
+
+const waitOnLock = async (lockFile) => {
+
+	while (await check(lockFile)) {
+		await sleep(100)
+	}
+
+}
+
+const mutexLock = async () => {
+
+
+	const dir = os.tmpdir();
+	const lockFile = `${dir}/${"agility-sync"}.mutex`
+	if (! fs.existsSync(lockFile)) {
+		fs.writeFileSync(lockFile, "agility-sync");
+	}
+
+	//THE LOCK IS ALREADY HELD - WAIT UP!
+	await waitOnLock(lockFile)
+
+	try {
+		return lockSync(lockFile)
+	} catch (err) {
+		if (`${err}`.indexOf("Lock file is already being held") !== -1) {
+
+			//this error happens when 2 processes try to get a lock at the EXACT same time (very rare)
+			await sleep(100)
+			await waitOnLock(lockFile)
+
+			try {
+				return lockSync(lockFile)
+			} catch (e2) {
+				if (`${err}`.indexOf("Lock file is already being held") !== -1) {
+
+					//this error happens when 2 processes try to get a lock at the EXACT same time (very rare)
+					await sleep(100)
+					await waitOnLock(lockFile)
+					return lockSync(lockFile)
+				}
+			}
+		}
+
+		throw Error("The mutex lock could not be obtained.")
+	}
+
+}
+
+
+
+
+
+module.exports = {
+	saveItem,
+	deleteItem,
+	mergeItemToList,
+	getItem,
+	clearItems,
+	mutexLock
+}