From c75ab41f97c95056fe4a0c3e8ba9bd3e279481e7 Mon Sep 17 00:00:00 2001 From: Laur Ivan Date: Fri, 8 Jul 2022 23:23:30 +0200 Subject: [PATCH] Import works. JS in async mode sucks. --- .env | 10 +- package-lock.json | 14 ++ package.json | 2 + src/Markdown.ts | 33 ++-- src/convertor/BlockConvertor.ts | 129 +++++++++----- src/convertor/DetailedConvertor.ts | 272 ----------------------------- src/convertor/images.ts | 23 ++- src/convertor/timer.ts | 1 + src/global.d.ts | 1 + 9 files changed, 155 insertions(+), 330 deletions(-) delete mode 100644 src/convertor/DetailedConvertor.ts create mode 100644 src/convertor/timer.ts diff --git a/.env b/.env index 9b6b979..06505b4 100644 --- a/.env +++ b/.env @@ -8,13 +8,17 @@ DOWNLOAD_IMAGES=false # Base URL # Final URL should be: https://www.laurivan.com/ -BASE_URL="http://localhost:2368" +BASE_URL="http://10.0.0.32:3000" # Ghost - related variables # -API_KEY="62ac6f5ab1479d0001082bd4:73341f843a5be78647c6f7e47d43d6cb09ec323df6d5706915df4685b3d46ce7" +API_KEY="62c86a0b59b1f400011aa9c3:8d711e07bc2a313b7012af5fc2f386434cc539c0f2efd1cad80f7c27476578b1" API_VERSION="v4.0" AUTHOR_EMAIL=laur.ivan@gmail.com # Location of head/hero images -HEAD_IMAGE_PATH="src/data/images/headers/" +HEAD_IMAGE_PATH="/home/laur/dev/ghost-dev/gatsby.laurivan.com/static/images/headers/" + +# Location of blogs +# BLOGS_PATH="/home/laur/dev/ghost-dev/gatsby.laurivan.com/archive/posts/" +BLOGS_PATH="/home/laur/dev/ghost-dev/gatsby.laurivan.com/archive/posts" diff --git a/package-lock.json b/package-lock.json index 8a018a2..85b2652 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@tryghost/admin-api": "^1.13.0", "@types/markdown-it": "^12.2.3", "axios": "^0.27.2", + "dive": "^0.5.0", "dotenv": "^16.0.1", "fs": "^0.0.1-security", "jsonwebtoken": "^8.5.1", @@ -155,6 +156,14 @@ "node": ">=0.3.1" } }, + "node_modules/dive": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/dive/-/dive-0.5.0.tgz", + "integrity": "sha512-T46KS5Qo6lYEx2nwGjh+VE36YEr2uJoXmrkZS7Skbl/LKowrV+OzzRTLcHIE1P38LAhpJOprYX5ysV9Rm3DLKw==", + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/dotenv": { "version": "16.0.1", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.1.tgz", @@ -652,6 +661,11 @@ "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", "dev": true }, + "dive": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/dive/-/dive-0.5.0.tgz", + "integrity": "sha512-T46KS5Qo6lYEx2nwGjh+VE36YEr2uJoXmrkZS7Skbl/LKowrV+OzzRTLcHIE1P38LAhpJOprYX5ysV9Rm3DLKw==" + }, "dotenv": { "version": "16.0.1", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.1.tgz", diff --git a/package.json b/package.json index f1759ca..ada59b1 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "sg": "npm run build && node ./dist/GhostAPI.js", "mm": "npm run build && node ./dist/MakeMobiledoc.js", "md": "npm run build && node ./dist/Markdown.js", + "mdq": "node ./dist/Markdown.js", "test": "echo \"Error: no test specified\" && exit 1" }, "author": { @@ -33,6 +34,7 @@ "@tryghost/admin-api": "^1.13.0", "@types/markdown-it": "^12.2.3", "axios": "^0.27.2", + "dive": "^0.5.0", "dotenv": "^16.0.1", "fs": "^0.0.1-security", "jsonwebtoken": "^8.5.1", diff --git a/src/Markdown.ts b/src/Markdown.ts index 180f127..98e10a6 100644 --- a/src/Markdown.ts +++ b/src/Markdown.ts @@ -2,8 +2,11 @@ * Load a file, parse it through markdonw-it and display its components */ import * as dotenv from "dotenv"; +import dive from "dive"; //import { ImageProcessor } from "./convertor/ImageProcessor"; import { BlockConvertor as Convertor } from "./convertor/BlockConvertor"; +import path from "node:path"; +import { timer } from "./convertor/timer"; const GhostAdminAPI = require("@tryghost/admin-api"); // Init config @@ -16,20 +19,24 @@ const api = new GhostAdminAPI({ key: process.env.API_KEY, version: process.env.API_VERSION, }); -let convertor = new Convertor( - "./src/data/blog/2019-10-31-ie-11-angular-compatibility.md" -); -//let imageProcessor = new ImageProcessor(); -async function main() { - //await imageProcessor.processImages(convertor, api, "./temp"); +async function main(filename: string) { + let convertor = new Convertor(filename); await convertor.process(api); + await timer(1000 * 1); } -main() - .then(() => { - console.log("OK"); - }) - .catch((r: any) => { - throw new Error(r); - }); +const extensions = [".markdown", ".md"]; +dive("" + process.env.BLOGS_PATH, (err: any, file: any) => { + if (err) throw err; + let ext = path.extname(file).toLowerCase(); + if (extensions.includes(ext)) { + main(file) + .then(() => { + console.log("OK"); + }) + .catch((r: any) => { + throw r; + }); + } +}); diff --git a/src/convertor/BlockConvertor.ts b/src/convertor/BlockConvertor.ts index 0483e52..ed7644c 100644 --- a/src/convertor/BlockConvertor.ts +++ b/src/convertor/BlockConvertor.ts @@ -1,14 +1,31 @@ import { PostNodeBuilder, Renderer } from "mobiledoc-kit"; import fs from "fs"; import metadataParser from "markdown-yaml-metadata-parser"; -import { Convertor } from "./Convertor"; +//import { Convertor } from "./Convertor"; //import { StackedImage } from "./ImageProcessor"; import path from "node:path"; import { downloadImagefromURL, uploadToGhost } from "./images"; +import axios from "axios"; + +const urlExist = async (url: string) => { + if (typeof url !== "string") { + throw new TypeError(`Expected a string, got ${typeof url}`); + } + + const response = await axios.head(url).catch(() => {}); + + return ( + response !== undefined && (response.status < 400 || response.status >= 500) + ); +}; const renderer: any = Renderer; -export class BlockConvertor implements Convertor { +if (renderer) { + console.log("Renderer enabled"); +} + +export class BlockConvertor { builder: any = new PostNodeBuilder(); filename: string; // The markdown bits (preamble and content) @@ -48,38 +65,63 @@ export class BlockConvertor implements Convertor { return this; } + /** + * Process the file + * + * @param api the ghost API object + * @returns Nothing really. + */ public async process(api: any) { - await this.processImages(api, "./temp"); + // Check if the entry is already there + let exists = await urlExist( + process.env.BASE_URL + "/" + this.metadata.slug + ); + console.log(`process: ${this.filename} - ${exists}`); + if (exists) return; + + let buffer = await this.processImages(api, "./temp"); + //console.log(buffer); + let markdownSection = this.builder.createCardSection("markdown", { - markdown: this.content, + markdown: buffer, }); let post = this.builder.createPost([markdownSection]); + let isDraft = + "draft" in this.metadata ? (this.metadata["draft"] ? true : false) : true; + let result = JSON.stringify({ title: this.metadata.title, slug: this.metadata.slug, mobiledoc: `${JSON.stringify(renderer.render(post, "0.3.1"))}`, - status: - "draft" in this.metadata - ? this.metadata["draft"] - ? "draft" - : "published" - : "draft", - visibility: "public", + status: "published", + visibility: isDraft ? "paid" : "public", created_at: this.metadata["date"], updated_at: this.metadata["date"], published_at: this.metadata["date"], tags: this.metadata["tags"], author: process.env.AUTHOR_EMAIL, - feature_image: this.featureImage, + feature_image: + this.featureImage === "@@@undefined" + ? "/media/generic.jpg" + : this.featureImage, + // feature_image_alt: this.metadata["cover"], + // feature_image_caption: this.metadata["cover"], }); - console.log(JSON.stringify(this.metadata)); - console.log(JSON.stringify(result)); - if (process.env.PUSH_POSTS === "true") { - await api.posts.add(JSON.parse(result)); + let retries = 0; + let pass = true; + while (retries < 5 && pass) { + await api.posts + .add(JSON.parse(result)) + .then(() => (pass = false)) + .catch((r: any) => { + console.log(JSON.stringify(r)); + retries++; + }); + } } else { console.log( "Env PUSH_POST is configured so no posts are saved to the ghost instance." @@ -94,18 +136,16 @@ export class BlockConvertor implements Convertor { * @param api the ghost api reference * @param basepath the path where to save images hosted online */ - public async processImages(api: any, basepath: string) { + public async processImages(api: any, basepath: string): Promise { // collect images in an array //let images: Record = {}; const imageRegex = /!\[[^\]]*\]\((.*?)\s*("(?:.*[^"])")?\s*\)/g; - //let images: string[] = []; - //let m: any; - [ - ...this.content.matchAll(imageRegex), - ["", "@@@" + this.metadata.cover, ""], - ].forEach(async (item) => { - console.log(JSON.stringify(item)); + let tempbuffer = this.content; + for (const item of [ + ...tempbuffer.matchAll(imageRegex), + ["", "@@@" + this.metadata.cover, ""], + ]) { let url = item[1]; let imageURL = url; let imagePath = url; @@ -121,27 +161,34 @@ export class BlockConvertor implements Convertor { // Download images locally if not already if (url.toLowerCase().startsWith("http")) { // Download the image locally - let localpath = path.join(basepath, path.basename(url)); + localpath = path.join(basepath, path.basename(url)); await downloadImagefromURL(url, localpath); } // Upload the image to ghost - if (api) { - let reference = await uploadToGhost(api, localpath); - imageURL = reference.url; + if (imagePath !== "@@@undefined") { + if (api) { + let reference = await uploadToGhost(api, localpath); + imageURL = reference.url.replace("" + process.env.BASE_URL, ""); + } + + // Set the feature image if any + if (imagePath.startsWith("@@@")) { + this.setFeatureImage(imageURL); + } else { + // replace the image string with the new URL + console.log( + " replace", + imagePath, + "with", + imageURL, + "for", + this.filename + ); + tempbuffer = tempbuffer.replaceAll(imagePath, imageURL); + } } - - // replace the image string with the new URL - console.log("replace", imagePath, " with ", imageURL); - this.content = this.content.replaceAll(imagePath, imageURL); - - // Set the feature image if any - if (imagePath.startsWith("@@@")) { - this.setFeatureImage(imageURL); - } - - // debug: show the updated content if necessary - // console.log(this.content); - }); + } + return tempbuffer; } } diff --git a/src/convertor/DetailedConvertor.ts b/src/convertor/DetailedConvertor.ts deleted file mode 100644 index a5ae493..0000000 --- a/src/convertor/DetailedConvertor.ts +++ /dev/null @@ -1,272 +0,0 @@ -import { PostNodeBuilder, Renderer } from "mobiledoc-kit"; -import fs from "fs"; -import MarkdownIt from "markdown-it"; -import Token from "markdown-it/lib/token"; -import metadataParser from "markdown-yaml-metadata-parser"; -import { Convertor } from "./Convertor"; - -const debugFence = false; -const debugInline = true; -const debugBlock = false; -const debugHeader = false; - -interface Stack { - tag: string; - meta: any; - token: Token; -} - -const renderer: any = Renderer; - -const predefinedBlocks = [ - "blockquote", - "table", - "bullet_list", - "ordered_list", - "paragraph", -]; - -const headerAsMarkdown = true; - -export class DetailedConvertor implements Convertor { - markups: Record = {}; - markers: Record = {}; - - sections: any[] = []; - builder: any = new PostNodeBuilder(); - filename: string; - - // The markdown bits (preamble and content) - // - metadata = {}; - content = ""; - - // The line-by-line split of the markdown - // - lines: string[] = []; - tokens: Token[] = []; - - /** - * Constructor - * - * @param filename the file name - */ - constructor(filename: string) { - this.filename = filename; - this.initialize(filename); - } - setFeatureImage(url: string): void { - throw new Error("Method not implemented.", url); - } - - /** - * Initialize/reset the convertor class. - * - * @param filename The file to be loaded - */ - public initialize(filename: string): DetailedConvertor { - this.builder = new PostNodeBuilder(); - let content = fs.readFileSync(filename, { - encoding: "utf8", - flag: "r", - }); - const parsed = metadataParser(content); - this.metadata = parsed.metadata; - this.content = parsed.content; - - // Split the lines - // - this.lines = this.content.split("\n"); - - const markdownProcessor = MarkdownIt("commonmark"); - this.tokens = markdownProcessor.parse(this.content, { references: {} }); - return this; - } - - public process(): string { - if (this.metadata === undefined && this.content == "") { - throw new Error( - "Convertor not initialised. Please use initalize(...) first" - ); - } - - let stack: Stack[] = []; - let blockNesting = 0; - - // Loop through tokens - this.tokens.forEach((value: Token) => { - let components: string[] = value.type.split("_"); - let action: string | undefined = components.pop(); - let name = components.join("_"); - console.log("BEGIN: ", value.tag, value.type); - switch (action) { - case "open": - if (name in predefinedBlocks) blockNesting++; - stack.push({ - tag: value.tag, - meta: value.meta, - token: value, - } as never); - break; - case "close": - if (name in predefinedBlocks) blockNesting--; - stack.pop(); - break; - case "fence": - case "block": - if (blockNesting == 0) { - this.sections.push(this.processFence(value)); - } - break; - case "inline": - /* - if (blockNesting == 0) { - let t: string = ""; - let m: string = ""; - - stack.forEach((value) => { - t += "-" + value.tag; - m += ">" + JSON.stringify(value.meta); - }); - console.log(t, m, value.content); - } - */ - break; - // TODO: Make the 'FENCE' tag too! - default: - console.log("ERROR", components, action); - //throw new Error("Con't know tag" + action); - } - if (value.level == 0) { - switch (name) { - case "table": - this.processBlock(value); - break; - case "heading": - //this.sections.push(this.processHeader(value)); - break; - default: - this.processBlock(value); - } - } else if (blockNesting == 0 && action === "inline") { - if (debugInline) - console.log(value.level, stack[0].token.type, stack[0].tag); - switch (stack[0].token.type) { - case "heading_open": - this.sections.push(this.processHeader(stack, value)); - break; - default: - this.sections.push(this.processInline(stack, value)); - } - } - }); - - // Aggregate the sections into a new mobiledoc - let post = this.builder.createPost(this.sections); - let result = JSON.stringify({ - title: "A post", - mobiledoc: `${JSON.stringify(renderer.render(post, "0.3.1"))}`, - status: "draft", - author: "laur.ivan@gmail.com", - }); - console.log("BLOG ENTRY------------------------------------"); - console.log(JSON.stringify(renderer.render(post, "0.3.1"))); - console.log("BLOG ENTRY------------------------------------"); - return result; - } - - private processInline(stack: Stack[], t: Token) { - if (debugInline) { - console.log("processInline:", stack.length, JSON.stringify(stack[0])); - } - // build the header as a markdown card - let tbuf: string[] = []; - - if (t.map != null) - for (let i = t.map[0]; i < t.map[1]; i++) tbuf.push(this.lines[i]); - let markdown = this.map2markdown(t.map); - - if (debugInline) { - console.log("processInline:", JSON.stringify(markdown)); - } - - let result = this.builder.createCardSection("markdown", { - markdown: markdown, - }); - return result; - } - - /** - * Get the string off lines range - * - * @param map the [lineStart, lineEnd] array - * @returns a sting containing the lines - */ - private map2markdown(map: [number, number] | null): string { - let tbuf: string[] = []; - if (map != null) - for (let i = map[0]; i < map[1]; i++) tbuf.push(this.lines[i]); - return tbuf.join("\n"); - } - - /** - * Process a fence (or code block) - * - * @param t the token of the fence - * @returns a markdown card - */ - private processFence(t: Token): any { - let result; - - if (debugFence) { - console.log("processFence:", console.log(JSON.stringify(t))); - } - - // build the fence as a markdown card - let markdown = this.map2markdown(t.map); - - if (debugFence) { - console.log("processFence:", console.log(JSON.stringify(markdown))); - } - - result = this.builder.createCardSection("markdown", { - markdown: markdown, - }); - return result; - } - - private processBlock(t: Token): any { - let markdown = this.map2markdown(t.map); - if (debugBlock) { - console.log("processBlock", JSON.stringify(t)); - } - - return { - content: markdown, - token: t, - }; - } - - private processHeader(stack: Stack[], t: Token): any { - if (debugHeader) { - console.log("processHeader", JSON.stringify(stack[0].token)); - } - - let result: any = undefined; - - if (!headerAsMarkdown) { - // Build the header as a tag - let marker = this.builder.createMarker(t.content, []); - result = this.builder.createMarkupSection(stack[0].tag, [marker]); - } else { - // build the header as a markdown card - let markdown = this.map2markdown(t.map); - - result = this.builder.createCardSection("markdown", { - markdown: markdown, - }); - } - - return result; - } -} diff --git a/src/convertor/images.ts b/src/convertor/images.ts index 4db645e..04a1d0e 100644 --- a/src/convertor/images.ts +++ b/src/convertor/images.ts @@ -5,6 +5,7 @@ import path from "path"; import { Blob } from "buffer"; import stream from "stream"; +//import { timer } from "./timer"; const Stream = stream.Transform; @@ -15,6 +16,7 @@ const Stream = stream.Transform; */ export async function downloadImagefromURL(url: string, filename: string) { let client: any = http; + let result = 200; if (url.toString().indexOf("https") === 0) { client = https; @@ -23,6 +25,7 @@ export async function downloadImagefromURL(url: string, filename: string) { client .request(url, function (response: any) { var data = new Stream(); + result = response.statusCode; response.on("data", function (chunk: any) { data.push(chunk); @@ -32,6 +35,7 @@ export async function downloadImagefromURL(url: string, filename: string) { }); }) .end(); + return result; } interface Image { @@ -60,6 +64,23 @@ export async function uploadToGhost( file: filename, purpose: "image", }; - let result = await api.images.upload(image); + //console.log("Upload: [%s]", filename); + // await timer(500 * 1); + + let retries = 0; + let pass = true; + let result: any; + while (retries < 5 && pass) { + await api.images + .upload(image) + .then((r: any) => { + result = r; + pass = false; + }) + .catch((r: any) => { + console.log(JSON.stringify(r)); + retries++; + }); + } return result; } diff --git a/src/convertor/timer.ts b/src/convertor/timer.ts new file mode 100644 index 0000000..7b39f42 --- /dev/null +++ b/src/convertor/timer.ts @@ -0,0 +1 @@ +export const timer = (ms: number) => new Promise((res) => setTimeout(res, ms)); diff --git a/src/global.d.ts b/src/global.d.ts index 43a0b8f..fdef4a0 100644 --- a/src/global.d.ts +++ b/src/global.d.ts @@ -2,3 +2,4 @@ declare module "mobiledoc-kit"; declare module "mobiledoc-kit/renderers/mobiledoc"; declare module "markdown-it-github-preamble"; declare module "markdown-yaml-metadata-parser"; +declare module "dive";