laur/markdown2ghost
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Import works.

Browse Source 
JS in async mode sucks.
This commit is contained in:
Laur Ivan 2022-07-08 23:23:30 +02:00
parent 4e95c02b6f
commit c75ab41f97
9 changed files with 155 additions and 330 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.env
View File

@ -8,13 +8,17 @@ DOWNLOAD_IMAGES=false
# Base URL
# Final URL should be: https://www.laurivan.com/
BASE_URL="http://localhost:2368"
BASE_URL="http://10.0.0.32:3000"
# Ghost - related variables
#
API_KEY="62ac6f5ab1479d0001082bd4:73341f843a5be78647c6f7e47d43d6cb09ec323df6d5706915df4685b3d46ce7"
API_KEY="62c86a0b59b1f400011aa9c3:8d711e07bc2a313b7012af5fc2f386434cc539c0f2efd1cad80f7c27476578b1"
API_VERSION="v4.0"
AUTHOR_EMAIL=laur.ivan@gmail.com
# Location of head/hero images
HEAD_IMAGE_PATH="src/data/images/headers/"
HEAD_IMAGE_PATH="/home/laur/dev/ghost-dev/gatsby.laurivan.com/static/images/headers/"
# Location of blogs
# BLOGS_PATH="/home/laur/dev/ghost-dev/gatsby.laurivan.com/archive/posts/"
BLOGS_PATH="/home/laur/dev/ghost-dev/gatsby.laurivan.com/archive/posts"

14
package-lock.json generated
View File

@ -12,6 +12,7 @@
"@tryghost/admin-api": "^1.13.0",
"@types/markdown-it": "^12.2.3",
"axios": "^0.27.2",
"dive": "^0.5.0",
"dotenv": "^16.0.1",
"fs": "^0.0.1-security",
"jsonwebtoken": "^8.5.1",
@ -155,6 +156,14 @@
"node": ">=0.3.1"
}
},
"node_modules/dive": {
"version": "0.5.0",
"resolved": "https://registry.npmjs.org/dive/-/dive-0.5.0.tgz",
"integrity": "sha512-T46KS5Qo6lYEx2nwGjh+VE36YEr2uJoXmrkZS7Skbl/LKowrV+OzzRTLcHIE1P38LAhpJOprYX5ysV9Rm3DLKw==",
"engines": {
"node": ">= 0.4.0"
}
},
"node_modules/dotenv": {
"version": "16.0.1",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.1.tgz",
@ -652,6 +661,11 @@
"integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
"dev": true
},
"dive": {
"version": "0.5.0",
"resolved": "https://registry.npmjs.org/dive/-/dive-0.5.0.tgz",
"integrity": "sha512-T46KS5Qo6lYEx2nwGjh+VE36YEr2uJoXmrkZS7Skbl/LKowrV+OzzRTLcHIE1P38LAhpJOprYX5ysV9Rm3DLKw=="
},
"dotenv": {
"version": "16.0.1",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.1.tgz",

2
package.json
View File

@ -17,6 +17,7 @@
"sg": "npm run build && node ./dist/GhostAPI.js",
"mm": "npm run build && node ./dist/MakeMobiledoc.js",
"md": "npm run build && node ./dist/Markdown.js",
"mdq": "node ./dist/Markdown.js",
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": {
@ -33,6 +34,7 @@
"@tryghost/admin-api": "^1.13.0",
"@types/markdown-it": "^12.2.3",
"axios": "^0.27.2",
"dive": "^0.5.0",
"dotenv": "^16.0.1",
"fs": "^0.0.1-security",
"jsonwebtoken": "^8.5.1",

33
src/Markdown.ts
View File

@ -2,8 +2,11 @@
* Load a file, parse it through markdonw-it and display its components
*/
import * as dotenv from "dotenv";
import dive from "dive";
//import { ImageProcessor } from "./convertor/ImageProcessor";
import { BlockConvertor as Convertor } from "./convertor/BlockConvertor";
import path from "node:path";
import { timer } from "./convertor/timer";
const GhostAdminAPI = require("@tryghost/admin-api");
// Init config
@ -16,20 +19,24 @@ const api = new GhostAdminAPI({
key: process.env.API_KEY,
version: process.env.API_VERSION,
});
let convertor = new Convertor(
"./src/data/blog/2019-10-31-ie-11-angular-compatibility.md"
);
//let imageProcessor = new ImageProcessor();
async function main() {
//await imageProcessor.processImages(convertor, api, "./temp");
async function main(filename: string) {
let convertor = new Convertor(filename);
await convertor.process(api);
await timer(1000 * 1);
}
main()
.then(() => {
console.log("OK");
})
.catch((r: any) => {
throw new Error(r);
});
const extensions = [".markdown", ".md"];
dive("" + process.env.BLOGS_PATH, (err: any, file: any) => {
if (err) throw err;
let ext = path.extname(file).toLowerCase();
if (extensions.includes(ext)) {
main(file)
.then(() => {
console.log("OK");
})
.catch((r: any) => {
throw r;
});
}
});

129
src/convertor/BlockConvertor.ts
View File

@ -1,14 +1,31 @@
import { PostNodeBuilder, Renderer } from "mobiledoc-kit";
import fs from "fs";
import metadataParser from "markdown-yaml-metadata-parser";
import { Convertor } from "./Convertor";
//import { Convertor } from "./Convertor";
//import { StackedImage } from "./ImageProcessor";
import path from "node:path";
import { downloadImagefromURL, uploadToGhost } from "./images";
import axios from "axios";
const urlExist = async (url: string) => {
if (typeof url !== "string") {
throw new TypeError(`Expected a string, got ${typeof url}`);
}
const response = await axios.head(url).catch(() => {});
return (
response !== undefined && (response.status < 400 || response.status >= 500)
);
};
const renderer: any = Renderer;
export class BlockConvertor implements Convertor {
if (renderer) {
console.log("Renderer enabled");
}
export class BlockConvertor {
builder: any = new PostNodeBuilder();
filename: string;
// The markdown bits (preamble and content)
@ -48,38 +65,63 @@ export class BlockConvertor implements Convertor {
return this;
}
/**
* Process the file
*
* @param api the ghost API object
* @returns Nothing really.
*/
public async process(api: any) {
await this.processImages(api, "./temp");
// Check if the entry is already there
let exists = await urlExist(
process.env.BASE_URL + "/" + this.metadata.slug
);
console.log(`process: ${this.filename} - ${exists}`);
if (exists) return;
let buffer = await this.processImages(api, "./temp");
//console.log(buffer);
let markdownSection = this.builder.createCardSection("markdown", {
markdown: this.content,
markdown: buffer,
});
let post = this.builder.createPost([markdownSection]);
let isDraft =
"draft" in this.metadata ? (this.metadata["draft"] ? true : false) : true;
let result = JSON.stringify({
title: this.metadata.title,
slug: this.metadata.slug,
mobiledoc: `${JSON.stringify(renderer.render(post, "0.3.1"))}`,
status:
"draft" in this.metadata
? this.metadata["draft"]
? "draft"
: "published"
: "draft",
visibility: "public",
status: "published",
visibility: isDraft ? "paid" : "public",
created_at: this.metadata["date"],
updated_at: this.metadata["date"],
published_at: this.metadata["date"],
tags: this.metadata["tags"],
author: process.env.AUTHOR_EMAIL,
feature_image: this.featureImage,
feature_image:
this.featureImage === "@@@undefined"
? "/media/generic.jpg"
: this.featureImage,
// feature_image_alt: this.metadata["cover"],
// feature_image_caption: this.metadata["cover"],
});
console.log(JSON.stringify(this.metadata));
console.log(JSON.stringify(result));
if (process.env.PUSH_POSTS === "true") {
await api.posts.add(JSON.parse(result));
let retries = 0;
let pass = true;
while (retries < 5 && pass) {
await api.posts
.add(JSON.parse(result))
.then(() => (pass = false))
.catch((r: any) => {
console.log(JSON.stringify(r));
retries++;
});
}
} else {
console.log(
"Env PUSH_POST is configured so no posts are saved to the ghost instance."
@ -94,18 +136,16 @@ export class BlockConvertor implements Convertor {
* @param api the ghost api reference
* @param basepath the path where to save images hosted online
*/
public async processImages(api: any, basepath: string) {
public async processImages(api: any, basepath: string): Promise<string> {
// collect images in an array
//let images: Record<string, StackedImage> = {};
const imageRegex = /!\[[^\]]*\]\((.*?)\s*("(?:.*[^"])")?\s*\)/g;
//let images: string[] = [];
//let m: any;
[
...this.content.matchAll(imageRegex),
["", "@@@" + this.metadata.cover, ""],
].forEach(async (item) => {
console.log(JSON.stringify(item));
let tempbuffer = this.content;
for (const item of [
...tempbuffer.matchAll(imageRegex),
["", "@@@" + this.metadata.cover, ""],
]) {
let url = item[1];
let imageURL = url;
let imagePath = url;
@ -121,27 +161,34 @@ export class BlockConvertor implements Convertor {
// Download images locally if not already
if (url.toLowerCase().startsWith("http")) {
// Download the image locally
let localpath = path.join(basepath, path.basename(url));
localpath = path.join(basepath, path.basename(url));
await downloadImagefromURL(url, localpath);
}
// Upload the image to ghost
if (api) {
let reference = await uploadToGhost(api, localpath);
imageURL = reference.url;
if (imagePath !== "@@@undefined") {
if (api) {
let reference = await uploadToGhost(api, localpath);
imageURL = reference.url.replace("" + process.env.BASE_URL, "");
}
// Set the feature image if any
if (imagePath.startsWith("@@@")) {
this.setFeatureImage(imageURL);
} else {
// replace the image string with the new URL
console.log(
" replace",
imagePath,
"with",
imageURL,
"for",
this.filename
);
tempbuffer = tempbuffer.replaceAll(imagePath, imageURL);
}
}
// replace the image string with the new URL
console.log("replace", imagePath, " with ", imageURL);
this.content = this.content.replaceAll(imagePath, imageURL);
// Set the feature image if any
if (imagePath.startsWith("@@@")) {
this.setFeatureImage(imageURL);
}
// debug: show the updated content if necessary
// console.log(this.content);
});
}
return tempbuffer;
}
}

272
src/convertor/DetailedConvertor.ts
View File

@ -1,272 +0,0 @@
import { PostNodeBuilder, Renderer } from "mobiledoc-kit";
import fs from "fs";
import MarkdownIt from "markdown-it";
import Token from "markdown-it/lib/token";
import metadataParser from "markdown-yaml-metadata-parser";
import { Convertor } from "./Convertor";
const debugFence = false;
const debugInline = true;
const debugBlock = false;
const debugHeader = false;
interface Stack {
tag: string;
meta: any;
token: Token;
}
const renderer: any = Renderer;
const predefinedBlocks = [
"blockquote",
"table",
"bullet_list",
"ordered_list",
"paragraph",
];
const headerAsMarkdown = true;
export class DetailedConvertor implements Convertor {
markups: Record<string, any> = {};
markers: Record<string, any> = {};
sections: any[] = [];
builder: any = new PostNodeBuilder();
filename: string;
// The markdown bits (preamble and content)
//
metadata = {};
content = "";
// The line-by-line split of the markdown
//
lines: string[] = [];
tokens: Token[] = [];
/**
* Constructor
*
* @param filename the file name
*/
constructor(filename: string) {
this.filename = filename;
this.initialize(filename);
}
setFeatureImage(url: string): void {
throw new Error("Method not implemented.", url);
}
/**
* Initialize/reset the convertor class.
*
* @param filename The file to be loaded
*/
public initialize(filename: string): DetailedConvertor {
this.builder = new PostNodeBuilder();
let content = fs.readFileSync(filename, {
encoding: "utf8",
flag: "r",
});
const parsed = metadataParser(content);
this.metadata = parsed.metadata;
this.content = parsed.content;
// Split the lines
//
this.lines = this.content.split("\n");
const markdownProcessor = MarkdownIt("commonmark");
this.tokens = markdownProcessor.parse(this.content, { references: {} });
return this;
}
public process(): string {
if (this.metadata === undefined && this.content == "") {
throw new Error(
"Convertor not initialised. Please use initalize(...) first"
);
}
let stack: Stack[] = [];
let blockNesting = 0;
// Loop through tokens
this.tokens.forEach((value: Token) => {
let components: string[] = value.type.split("_");
let action: string | undefined = components.pop();
let name = components.join("_");
console.log("BEGIN: ", value.tag, value.type);
switch (action) {
case "open":
if (name in predefinedBlocks) blockNesting++;
stack.push({
tag: value.tag,
meta: value.meta,
token: value,
} as never);
break;
case "close":
if (name in predefinedBlocks) blockNesting--;
stack.pop();
break;
case "fence":
case "block":
if (blockNesting == 0) {
this.sections.push(this.processFence(value));
}
break;
case "inline":
/*
if (blockNesting == 0) {
let t: string = "";
let m: string = "";
stack.forEach((value) => {
t += "-" + value.tag;
m += ">" + JSON.stringify(value.meta);
});
console.log(t, m, value.content);
}
*/
break;
// TODO: Make the 'FENCE' tag too!
default:
console.log("ERROR", components, action);
//throw new Error("Con't know tag" + action);
}
if (value.level == 0) {
switch (name) {
case "table":
this.processBlock(value);
break;
case "heading":
//this.sections.push(this.processHeader(value));
break;
default:
this.processBlock(value);
}
} else if (blockNesting == 0 && action === "inline") {
if (debugInline)
console.log(value.level, stack[0].token.type, stack[0].tag);
switch (stack[0].token.type) {
case "heading_open":
this.sections.push(this.processHeader(stack, value));
break;
default:
this.sections.push(this.processInline(stack, value));
}
}
});
// Aggregate the sections into a new mobiledoc
let post = this.builder.createPost(this.sections);
let result = JSON.stringify({
title: "A post",
mobiledoc: `${JSON.stringify(renderer.render(post, "0.3.1"))}`,
status: "draft",
author: "laur.ivan@gmail.com",
});
console.log("BLOG ENTRY------------------------------------");
console.log(JSON.stringify(renderer.render(post, "0.3.1")));
console.log("BLOG ENTRY------------------------------------");
return result;
}
private processInline(stack: Stack[], t: Token) {
if (debugInline) {
console.log("processInline:", stack.length, JSON.stringify(stack[0]));
}
// build the header as a markdown card
let tbuf: string[] = [];
if (t.map != null)
for (let i = t.map[0]; i < t.map[1]; i++) tbuf.push(this.lines[i]);
let markdown = this.map2markdown(t.map);
if (debugInline) {
console.log("processInline:", JSON.stringify(markdown));
}
let result = this.builder.createCardSection("markdown", {
markdown: markdown,
});
return result;
}
/**
* Get the string off lines range
*
* @param map the [lineStart, lineEnd] array
* @returns a sting containing the lines
*/
private map2markdown(map: [number, number] | null): string {
let tbuf: string[] = [];
if (map != null)
for (let i = map[0]; i < map[1]; i++) tbuf.push(this.lines[i]);
return tbuf.join("\n");
}
/**
* Process a fence (or code block)
*
* @param t the token of the fence
* @returns a markdown card
*/
private processFence(t: Token): any {
let result;
if (debugFence) {
console.log("processFence:", console.log(JSON.stringify(t)));
}
// build the fence as a markdown card
let markdown = this.map2markdown(t.map);
if (debugFence) {
console.log("processFence:", console.log(JSON.stringify(markdown)));
}
result = this.builder.createCardSection("markdown", {
markdown: markdown,
});
return result;
}
private processBlock(t: Token): any {
let markdown = this.map2markdown(t.map);
if (debugBlock) {
console.log("processBlock", JSON.stringify(t));
}
return {
content: markdown,
token: t,
};
}
private processHeader(stack: Stack[], t: Token): any {
if (debugHeader) {
console.log("processHeader", JSON.stringify(stack[0].token));
}
let result: any = undefined;
if (!headerAsMarkdown) {
// Build the header as a <Hx /> tag
let marker = this.builder.createMarker(t.content, []);
result = this.builder.createMarkupSection(stack[0].tag, [marker]);
} else {
// build the header as a markdown card
let markdown = this.map2markdown(t.map);
result = this.builder.createCardSection("markdown", {
markdown: markdown,
});
}
return result;
}
}

23
src/convertor/images.ts
View File

@ -5,6 +5,7 @@ import path from "path";
import { Blob } from "buffer";
import stream from "stream";
//import { timer } from "./timer";
const Stream = stream.Transform;
@ -15,6 +16,7 @@ const Stream = stream.Transform;
*/
export async function downloadImagefromURL(url: string, filename: string) {
let client: any = http;
let result = 200;
if (url.toString().indexOf("https") === 0) {
client = https;
@ -23,6 +25,7 @@ export async function downloadImagefromURL(url: string, filename: string) {
client
.request(url, function (response: any) {
var data = new Stream();
result = response.statusCode;
response.on("data", function (chunk: any) {
data.push(chunk);
@ -32,6 +35,7 @@ export async function downloadImagefromURL(url: string, filename: string) {
});
})
.end();
return result;
}
interface Image {
@ -60,6 +64,23 @@ export async function uploadToGhost(
file: filename,
purpose: "image",
};
let result = await api.images.upload(image);
//console.log("Upload: [%s]", filename);
// await timer(500 * 1);
let retries = 0;
let pass = true;
let result: any;
while (retries < 5 && pass) {
await api.images
.upload(image)
.then((r: any) => {
result = r;
pass = false;
})
.catch((r: any) => {
console.log(JSON.stringify(r));
retries++;
});
}
return result;
}

1
src/convertor/timer.ts Normal file
View File

@ -0,0 +1 @@
export const timer = (ms: number) => new Promise((res) => setTimeout(res, ms));

1
src/global.d.ts vendored
View File

@ -2,3 +2,4 @@ declare module "mobiledoc-kit";
declare module "mobiledoc-kit/renderers/mobiledoc";
declare module "markdown-it-github-preamble";
declare module "markdown-yaml-metadata-parser";
declare module "dive";