Import works.

JS in async mode sucks.

.env

@@ -8,13 +8,17 @@ DOWNLOAD_IMAGES=false
 
 # Base URL
 # Final URL should be: https://www.laurivan.com/
-BASE_URL="http://localhost:2368"
+BASE_URL="http://10.0.0.32:3000"
 
 # Ghost - related variables
 #
-API_KEY="62ac6f5ab1479d0001082bd4:73341f843a5be78647c6f7e47d43d6cb09ec323df6d5706915df4685b3d46ce7"
+API_KEY="62c86a0b59b1f400011aa9c3:8d711e07bc2a313b7012af5fc2f386434cc539c0f2efd1cad80f7c27476578b1"
 API_VERSION="v4.0"
 AUTHOR_EMAIL=laur.ivan@gmail.com
 
 # Location of head/hero images
-HEAD_IMAGE_PATH="src/data/images/headers/"
+HEAD_IMAGE_PATH="/home/laur/dev/ghost-dev/gatsby.laurivan.com/static/images/headers/"
+
+# Location of blogs
+# BLOGS_PATH="/home/laur/dev/ghost-dev/gatsby.laurivan.com/archive/posts/"
+BLOGS_PATH="/home/laur/dev/ghost-dev/gatsby.laurivan.com/archive/posts"

package-lock.json

@@ -12,6 +12,7 @@
         "@tryghost/admin-api": "^1.13.0",
         "@types/markdown-it": "^12.2.3",
         "axios": "^0.27.2",
+        "dive": "^0.5.0",
         "dotenv": "^16.0.1",
         "fs": "^0.0.1-security",
         "jsonwebtoken": "^8.5.1",
@@ -155,6 +156,14 @@
         "node": ">=0.3.1"
       }
     },
+    "node_modules/dive": {
+      "version": "0.5.0",
+      "resolved": "https://registry.npmjs.org/dive/-/dive-0.5.0.tgz",
+      "integrity": "sha512-T46KS5Qo6lYEx2nwGjh+VE36YEr2uJoXmrkZS7Skbl/LKowrV+OzzRTLcHIE1P38LAhpJOprYX5ysV9Rm3DLKw==",
+      "engines": {
+        "node": ">= 0.4.0"
+      }
+    },
     "node_modules/dotenv": {
       "version": "16.0.1",
       "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.1.tgz",
@@ -652,6 +661,11 @@
       "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
       "dev": true
     },
+    "dive": {
+      "version": "0.5.0",
+      "resolved": "https://registry.npmjs.org/dive/-/dive-0.5.0.tgz",
+      "integrity": "sha512-T46KS5Qo6lYEx2nwGjh+VE36YEr2uJoXmrkZS7Skbl/LKowrV+OzzRTLcHIE1P38LAhpJOprYX5ysV9Rm3DLKw=="
+    },
     "dotenv": {
       "version": "16.0.1",
       "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.1.tgz",

package.json

@@ -17,6 +17,7 @@
     "sg": "npm run build && node ./dist/GhostAPI.js",
     "mm": "npm run build && node ./dist/MakeMobiledoc.js",
     "md": "npm run build && node ./dist/Markdown.js",
+    "mdq": "node ./dist/Markdown.js",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "author": {
@@ -33,6 +34,7 @@
     "@tryghost/admin-api": "^1.13.0",
     "@types/markdown-it": "^12.2.3",
     "axios": "^0.27.2",
+    "dive": "^0.5.0",
     "dotenv": "^16.0.1",
     "fs": "^0.0.1-security",
     "jsonwebtoken": "^8.5.1",

src/Markdown.ts

@@ -2,8 +2,11 @@
  * Load a file, parse it through markdonw-it and display its components
  */
 import * as dotenv from "dotenv";
+import dive from "dive";
 //import { ImageProcessor } from "./convertor/ImageProcessor";
 import { BlockConvertor as Convertor } from "./convertor/BlockConvertor";
+import path from "node:path";
+import { timer } from "./convertor/timer";
 const GhostAdminAPI = require("@tryghost/admin-api");
 
 // Init config
@@ -16,20 +19,24 @@ const api = new GhostAdminAPI({
   key: process.env.API_KEY,
   version: process.env.API_VERSION,
 });
-let convertor = new Convertor(
-  "./src/data/blog/2019-10-31-ie-11-angular-compatibility.md"
-);
-//let imageProcessor = new ImageProcessor();
 
-async function main() {
+async function main(filename: string) {
-  //await imageProcessor.processImages(convertor, api, "./temp");
+  let convertor = new Convertor(filename);
   await convertor.process(api);
+  await timer(1000 * 1);
 }
 
-main()
+const extensions = [".markdown", ".md"];
-  .then(() => {
+dive("" + process.env.BLOGS_PATH, (err: any, file: any) => {
-    console.log("OK");
+  if (err) throw err;
-  })
+  let ext = path.extname(file).toLowerCase();
-  .catch((r: any) => {
+  if (extensions.includes(ext)) {
-    throw new Error(r);
+    main(file)
-  });
+      .then(() => {
+        console.log("OK");
+      })
+      .catch((r: any) => {
+        throw r;
+      });
+  }
+});

src/convertor/BlockConvertor.ts

@@ -1,14 +1,31 @@
 import { PostNodeBuilder, Renderer } from "mobiledoc-kit";
 import fs from "fs";
 import metadataParser from "markdown-yaml-metadata-parser";
-import { Convertor } from "./Convertor";
+//import { Convertor } from "./Convertor";
 //import { StackedImage } from "./ImageProcessor";
 import path from "node:path";
 import { downloadImagefromURL, uploadToGhost } from "./images";
+import axios from "axios";
+
+const urlExist = async (url: string) => {
+  if (typeof url !== "string") {
+    throw new TypeError(`Expected a string, got ${typeof url}`);
+  }
+
+  const response = await axios.head(url).catch(() => {});
+
+  return (
+    response !== undefined && (response.status < 400 || response.status >= 500)
+  );
+};
 
 const renderer: any = Renderer;
 
-export class BlockConvertor implements Convertor {
+if (renderer) {
+  console.log("Renderer enabled");
+}
+
+export class BlockConvertor {
   builder: any = new PostNodeBuilder();
   filename: string;
   // The markdown bits (preamble and content)
@@ -48,38 +65,63 @@ export class BlockConvertor implements Convertor {
     return this;
   }
 
+  /**
+   * Process the file
+   *
+   * @param api the ghost API object
+   * @returns Nothing really.
+   */
   public async process(api: any) {
-    await this.processImages(api, "./temp");
+    // Check if the entry is already there
+    let exists = await urlExist(
+      process.env.BASE_URL + "/" + this.metadata.slug
+    );
+    console.log(`process: ${this.filename} - ${exists}`);
+    if (exists) return;
+
+    let buffer = await this.processImages(api, "./temp");
+    //console.log(buffer);
+
     let markdownSection = this.builder.createCardSection("markdown", {
-      markdown: this.content,
+      markdown: buffer,
     });
 
     let post = this.builder.createPost([markdownSection]);
 
+    let isDraft =
+      "draft" in this.metadata ? (this.metadata["draft"] ? true : false) : true;
+
     let result = JSON.stringify({
       title: this.metadata.title,
       slug: this.metadata.slug,
       mobiledoc: `${JSON.stringify(renderer.render(post, "0.3.1"))}`,
-      status:
+      status: "published",
-        "draft" in this.metadata
+      visibility: isDraft ? "paid" : "public",
-          ? this.metadata["draft"]
-            ? "draft"
-            : "published"
-          : "draft",
-      visibility: "public",
       created_at: this.metadata["date"],
       updated_at: this.metadata["date"],
       published_at: this.metadata["date"],
       tags: this.metadata["tags"],
       author: process.env.AUTHOR_EMAIL,
-      feature_image: this.featureImage,
+      feature_image:
+        this.featureImage === "@@@undefined"
+          ? "/media/generic.jpg"
+          : this.featureImage,
+      // feature_image_alt: this.metadata["cover"],
+      // feature_image_caption: this.metadata["cover"],
     });
 
-    console.log(JSON.stringify(this.metadata));
-    console.log(JSON.stringify(result));
-
     if (process.env.PUSH_POSTS === "true") {
-      await api.posts.add(JSON.parse(result));
+      let retries = 0;
+      let pass = true;
+      while (retries < 5 && pass) {
+        await api.posts
+          .add(JSON.parse(result))
+          .then(() => (pass = false))
+          .catch((r: any) => {
+            console.log(JSON.stringify(r));
+            retries++;
+          });
+      }
     } else {
       console.log(
         "Env PUSH_POST is configured so no posts are saved to the ghost instance."
@@ -94,18 +136,16 @@ export class BlockConvertor implements Convertor {
    * @param api the ghost api reference
    * @param basepath the path where to save images hosted online
    */
-  public async processImages(api: any, basepath: string) {
+  public async processImages(api: any, basepath: string): Promise<string> {
     // collect images in an array
     //let images: Record<string, StackedImage> = {};
     const imageRegex = /!\[[^\]]*\]\((.*?)\s*("(?:.*[^"])")?\s*\)/g;
-    //let images: string[] = [];
+    let tempbuffer = this.content;
-    //let m: any;
-    [
-      ...this.content.matchAll(imageRegex),
-      ["", "@@@" + this.metadata.cover, ""],
-    ].forEach(async (item) => {
-      console.log(JSON.stringify(item));
 
+    for (const item of [
+      ...tempbuffer.matchAll(imageRegex),
+      ["", "@@@" + this.metadata.cover, ""],
+    ]) {
       let url = item[1];
       let imageURL = url;
       let imagePath = url;
@@ -121,27 +161,34 @@ export class BlockConvertor implements Convertor {
       // Download images locally if not already
       if (url.toLowerCase().startsWith("http")) {
         // Download the image locally
-        let localpath = path.join(basepath, path.basename(url));
+        localpath = path.join(basepath, path.basename(url));
         await downloadImagefromURL(url, localpath);
       }
 
       // Upload the image to ghost
-      if (api) {
+      if (imagePath !== "@@@undefined") {
-        let reference = await uploadToGhost(api, localpath);
+        if (api) {
-        imageURL = reference.url;
+          let reference = await uploadToGhost(api, localpath);
+          imageURL = reference.url.replace("" + process.env.BASE_URL, "");
+        }
+
+        // Set the feature image if any
+        if (imagePath.startsWith("@@@")) {
+          this.setFeatureImage(imageURL);
+        } else {
+          // replace the image string with the new URL
+          console.log(
+            "  replace",
+            imagePath,
+            "with",
+            imageURL,
+            "for",
+            this.filename
+          );
+          tempbuffer = tempbuffer.replaceAll(imagePath, imageURL);
+        }
       }
-
+    }
-      // replace the image string with the new URL
+    return tempbuffer;
-      console.log("replace", imagePath, " with ", imageURL);
-      this.content = this.content.replaceAll(imagePath, imageURL);
-
-      // Set the feature image if any
-      if (imagePath.startsWith("@@@")) {
-        this.setFeatureImage(imageURL);
-      }
-
-      // debug: show the updated content if necessary
-      // console.log(this.content);
-    });
   }
 }

src/convertor/DetailedConvertor.ts

@@ -1,272 +0,0 @@
-import { PostNodeBuilder, Renderer } from "mobiledoc-kit";
-import fs from "fs";
-import MarkdownIt from "markdown-it";
-import Token from "markdown-it/lib/token";
-import metadataParser from "markdown-yaml-metadata-parser";
-import { Convertor } from "./Convertor";
-
-const debugFence = false;
-const debugInline = true;
-const debugBlock = false;
-const debugHeader = false;
-
-interface Stack {
-  tag: string;
-  meta: any;
-  token: Token;
-}
-
-const renderer: any = Renderer;
-
-const predefinedBlocks = [
-  "blockquote",
-  "table",
-  "bullet_list",
-  "ordered_list",
-  "paragraph",
-];
-
-const headerAsMarkdown = true;
-
-export class DetailedConvertor implements Convertor {
-  markups: Record<string, any> = {};
-  markers: Record<string, any> = {};
-
-  sections: any[] = [];
-  builder: any = new PostNodeBuilder();
-  filename: string;
-
-  // The markdown bits (preamble and content)
-  //
-  metadata = {};
-  content = "";
-
-  // The line-by-line split of the markdown
-  //
-  lines: string[] = [];
-  tokens: Token[] = [];
-
-  /**
-   * Constructor
-   *
-   * @param filename the file name
-   */
-  constructor(filename: string) {
-    this.filename = filename;
-    this.initialize(filename);
-  }
-  setFeatureImage(url: string): void {
-    throw new Error("Method not implemented.", url);
-  }
-
-  /**
-   * Initialize/reset the convertor class.
-   *
-   * @param filename The file to be loaded
-   */
-  public initialize(filename: string): DetailedConvertor {
-    this.builder = new PostNodeBuilder();
-    let content = fs.readFileSync(filename, {
-      encoding: "utf8",
-      flag: "r",
-    });
-    const parsed = metadataParser(content);
-    this.metadata = parsed.metadata;
-    this.content = parsed.content;
-
-    // Split the lines
-    //
-    this.lines = this.content.split("\n");
-
-    const markdownProcessor = MarkdownIt("commonmark");
-    this.tokens = markdownProcessor.parse(this.content, { references: {} });
-    return this;
-  }
-
-  public process(): string {
-    if (this.metadata === undefined && this.content == "") {
-      throw new Error(
-        "Convertor not initialised. Please use initalize(...) first"
-      );
-    }
-
-    let stack: Stack[] = [];
-    let blockNesting = 0;
-
-    // Loop through tokens
-    this.tokens.forEach((value: Token) => {
-      let components: string[] = value.type.split("_");
-      let action: string | undefined = components.pop();
-      let name = components.join("_");
-      console.log("BEGIN: ", value.tag, value.type);
-      switch (action) {
-        case "open":
-          if (name in predefinedBlocks) blockNesting++;
-          stack.push({
-            tag: value.tag,
-            meta: value.meta,
-            token: value,
-          } as never);
-          break;
-        case "close":
-          if (name in predefinedBlocks) blockNesting--;
-          stack.pop();
-          break;
-        case "fence":
-        case "block":
-          if (blockNesting == 0) {
-            this.sections.push(this.processFence(value));
-          }
-          break;
-        case "inline":
-          /*
-          if (blockNesting == 0) {
-            let t: string = "";
-            let m: string = "";
-
-            stack.forEach((value) => {
-              t += "-" + value.tag;
-              m += ">" + JSON.stringify(value.meta);
-            });
-            console.log(t, m, value.content);
-          }
-          */
-          break;
-        // TODO: Make the 'FENCE' tag too!
-        default:
-          console.log("ERROR", components, action);
-        //throw new Error("Con't know tag" + action);
-      }
-      if (value.level == 0) {
-        switch (name) {
-          case "table":
-            this.processBlock(value);
-            break;
-          case "heading":
-            //this.sections.push(this.processHeader(value));
-            break;
-          default:
-            this.processBlock(value);
-        }
-      } else if (blockNesting == 0 && action === "inline") {
-        if (debugInline)
-          console.log(value.level, stack[0].token.type, stack[0].tag);
-        switch (stack[0].token.type) {
-          case "heading_open":
-            this.sections.push(this.processHeader(stack, value));
-            break;
-          default:
-            this.sections.push(this.processInline(stack, value));
-        }
-      }
-    });
-
-    // Aggregate the sections into a new mobiledoc
-    let post = this.builder.createPost(this.sections);
-    let result = JSON.stringify({
-      title: "A post",
-      mobiledoc: `${JSON.stringify(renderer.render(post, "0.3.1"))}`,
-      status: "draft",
-      author: "laur.ivan@gmail.com",
-    });
-    console.log("BLOG ENTRY------------------------------------");
-    console.log(JSON.stringify(renderer.render(post, "0.3.1")));
-    console.log("BLOG ENTRY------------------------------------");
-    return result;
-  }
-
-  private processInline(stack: Stack[], t: Token) {
-    if (debugInline) {
-      console.log("processInline:", stack.length, JSON.stringify(stack[0]));
-    }
-    // build the header as a markdown card
-    let tbuf: string[] = [];
-
-    if (t.map != null)
-      for (let i = t.map[0]; i < t.map[1]; i++) tbuf.push(this.lines[i]);
-    let markdown = this.map2markdown(t.map);
-
-    if (debugInline) {
-      console.log("processInline:", JSON.stringify(markdown));
-    }
-
-    let result = this.builder.createCardSection("markdown", {
-      markdown: markdown,
-    });
-    return result;
-  }
-
-  /**
-   * Get the string off lines range
-   *
-   * @param map the [lineStart, lineEnd] array
-   * @returns a sting containing the lines
-   */
-  private map2markdown(map: [number, number] | null): string {
-    let tbuf: string[] = [];
-    if (map != null)
-      for (let i = map[0]; i < map[1]; i++) tbuf.push(this.lines[i]);
-    return tbuf.join("\n");
-  }
-
-  /**
-   * Process a fence (or code block)
-   *
-   * @param t the token of the fence
-   * @returns a markdown card
-   */
-  private processFence(t: Token): any {
-    let result;
-
-    if (debugFence) {
-      console.log("processFence:", console.log(JSON.stringify(t)));
-    }
-
-    // build the fence as a markdown card
-    let markdown = this.map2markdown(t.map);
-
-    if (debugFence) {
-      console.log("processFence:", console.log(JSON.stringify(markdown)));
-    }
-
-    result = this.builder.createCardSection("markdown", {
-      markdown: markdown,
-    });
-    return result;
-  }
-
-  private processBlock(t: Token): any {
-    let markdown = this.map2markdown(t.map);
-    if (debugBlock) {
-      console.log("processBlock", JSON.stringify(t));
-    }
-
-    return {
-      content: markdown,
-      token: t,
-    };
-  }
-
-  private processHeader(stack: Stack[], t: Token): any {
-    if (debugHeader) {
-      console.log("processHeader", JSON.stringify(stack[0].token));
-    }
-
-    let result: any = undefined;
-
-    if (!headerAsMarkdown) {
-      // Build the header as a <Hx /> tag
-      let marker = this.builder.createMarker(t.content, []);
-      result = this.builder.createMarkupSection(stack[0].tag, [marker]);
-    } else {
-      // build the header as a markdown card
-      let markdown = this.map2markdown(t.map);
-
-      result = this.builder.createCardSection("markdown", {
-        markdown: markdown,
-      });
-    }
-
-    return result;
-  }
-}

src/convertor/images.ts

@@ -5,6 +5,7 @@ import path from "path";
 import { Blob } from "buffer";
 
 import stream from "stream";
+//import { timer } from "./timer";
 
 const Stream = stream.Transform;
 
@@ -15,6 +16,7 @@ const Stream = stream.Transform;
  */
 export async function downloadImagefromURL(url: string, filename: string) {
   let client: any = http;
+  let result = 200;
 
   if (url.toString().indexOf("https") === 0) {
     client = https;
@@ -23,6 +25,7 @@ export async function downloadImagefromURL(url: string, filename: string) {
   client
     .request(url, function (response: any) {
       var data = new Stream();
+      result = response.statusCode;
 
       response.on("data", function (chunk: any) {
         data.push(chunk);
@@ -32,6 +35,7 @@ export async function downloadImagefromURL(url: string, filename: string) {
       });
     })
     .end();
+  return result;
 }
 
 interface Image {
@@ -60,6 +64,23 @@ export async function uploadToGhost(
     file: filename,
     purpose: "image",
   };
-  let result = await api.images.upload(image);
+  //console.log("Upload: [%s]", filename);
+  // await timer(500 * 1);
+
+  let retries = 0;
+  let pass = true;
+  let result: any;
+  while (retries < 5 && pass) {
+    await api.images
+      .upload(image)
+      .then((r: any) => {
+        result = r;
+        pass = false;
+      })
+      .catch((r: any) => {
+        console.log(JSON.stringify(r));
+        retries++;
+      });
+  }
   return result;
 }

src/convertor/timer.ts

@@ -0,0 +1 @@
+export const timer = (ms: number) => new Promise((res) => setTimeout(res, ms));

src/global.d.ts

@@ -2,3 +2,4 @@ declare module "mobiledoc-kit";
 declare module "mobiledoc-kit/renderers/mobiledoc";
 declare module "markdown-it-github-preamble";
 declare module "markdown-yaml-metadata-parser";
+declare module "dive";