From 4db6e5f4fb2eccbd9b0b9390118d9ff2acfe5775 Mon Sep 17 00:00:00 2001 From: Elio Struyf Date: Wed, 4 Sep 2019 10:12:15 +0200 Subject: [PATCH] #4 - Added support for generating slugs --- README.md | 14 + package.json | 15 +- src/commands/Article.ts | 27 +- src/constants/charMap.ts | 276 ++++++++++++++ src/constants/settings.ts | 5 +- src/constants/stopwords-en.ts | 661 ++++++++++++++++++++++++++++++++++ src/extension.ts | 5 + src/helpers/ArticleHelper.ts | 60 +++ 8 files changed, 1060 insertions(+), 3 deletions(-) create mode 100644 src/constants/charMap.ts create mode 100644 src/constants/stopwords-en.ts diff --git a/README.md b/README.md index a6b3f357..f2431fca 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,20 @@ Update the `date` property of the current article/post/... to the current date & > **Optional**: if you want, you can specify the format of the date property by adding your own preference in your settings. Settings key: `frontMatter.taxonomy.dateFormat`. Check [date-fns formating](https://date-fns.org/v2.0.1/docs/format) for more information which patterns you can use. +**Front Matter: Generate slug based on article title** + +Generates a clean slug for your article. It removes known stop words, punctuations, and special characters. + +Example: +``` +title: Just a sample page with a title +slug: sample-page-title +``` + +If you want, you can also specify a prefix and suffix which can be added to the slug. Use the following settings to do this: `frontMatter.taxonomy.slugPrefix` and `frontMatter.taxonomy.slugSuffix`. By default both options are not provided and will not add anything to the slug. + +> **Info**: At the moment only English stop words are supported. + ## Where is the data stored? The tags and categories are stored in the project VSCode user settings. You can find them back under: `.vscode/settings.json`. diff --git a/package.json b/package.json index f0585dae..3f0a1079 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,8 @@ "onCommand:frontMatter.createCategory", "onCommand:frontMatter.exportTaxonomy", "onCommand:frontMatter.remap", - "onCommand:frontMatter.setDate" + "onCommand:frontMatter.setDate", + "onCommand:frontMatter.generateSlug" ], "main": "./dist/extension", "contributes": { @@ -62,6 +63,14 @@ "frontMatter.taxonomy.dateFormat": { "type": "string", "markdownDescription": "Specify the date format for your articles. Check [date-fns formating](https://date-fns.org/v2.0.1/docs/format) for more information." + }, + "frontMatter.taxonomy.slugPrefix": { + "type": "string", + "markdownDescription": "Specify a prefix for the slug" + }, + "frontMatter.taxonomy.slugSuffix": { + "type": "string", + "markdownDescription": "Specify a suffix for the slug" } } }, @@ -93,6 +102,10 @@ { "command": "frontMatter.setDate", "title": "Front Matter: Set current date" + }, + { + "command": "frontMatter.generateSlug", + "title": "Front Matter: Generate slug based on article title" } ] }, diff --git a/src/commands/Article.ts b/src/commands/Article.ts index e8fdab7e..f7fae8fe 100644 --- a/src/commands/Article.ts +++ b/src/commands/Article.ts @@ -1,6 +1,6 @@ import * as vscode from 'vscode'; import { TaxonomyType } from "../models"; -import { CONFIG_KEY, ACTION_TAXONOMY_TAGS, ACTION_TAXONOMY_CATEGORIES, ACTION_DATE_FORMAT, EXTENSION_NAME } from "../constants/settings"; +import { CONFIG_KEY, ACTION_TAXONOMY_TAGS, ACTION_TAXONOMY_CATEGORIES, ACTION_DATE_FORMAT, EXTENSION_NAME, ACTION_SLUG_PREFIX, ACTION_SLUG_SUFFIX } from "../constants/settings"; import { format } from "date-fns"; import { ArticleHelper, SettingsHelper } from '../helpers'; @@ -97,6 +97,31 @@ export class Article { } } + /** + * Generate the slug based on the article title + */ + public static generateSlug() { + const config = vscode.workspace.getConfiguration(CONFIG_KEY); + const prefix = config.get(ACTION_SLUG_PREFIX) as string; + const suffix = config.get(ACTION_SLUG_SUFFIX) as string; + const editor = vscode.window.activeTextEditor; + if (!editor) { + return; + } + + const article = ArticleHelper.getFrontMatter(editor); + if (!article || !article.data) { + return; + } + + const articleTitle: string = article.data["title"]; + const slug = ArticleHelper.createSlug(articleTitle); + if (slug) { + article.data["slug"] = `${prefix}${slug}${suffix}`; + ArticleHelper.update(editor, article); + } + } + /** * Toggle the page its draft mode */ diff --git a/src/constants/charMap.ts b/src/constants/charMap.ts new file mode 100644 index 00000000..9fb5922a --- /dev/null +++ b/src/constants/charMap.ts @@ -0,0 +1,276 @@ +export const charMap: { [character: string]: string } = { + // latin + "À": "A", + "Á": "A", + "Â": "A", + "Ã": "A", + "Ä": "A", + "Å": "A", + "Æ": "AE", + "Ç": "C", + "È": "E", + "É": "E", + "Ê": "E", + "Ë": "E", + "Ì": "I", + "Í": "I", + "Î": "I", + "Ï": "I", + "Ð": "D", + "Ñ": "N", + "Ò": "O", + "Ó": "O", + "Ô": "O", + "Õ": "O", + "Ö": "O", + "Ő": "O", + "Ø": "O", + "Ù": "U", + "Ú": "U", + "Û": "U", + "Ü": "U", + "Ű": "U", + "Ý": "Y", + "Þ": "TH", + "ß": "ss", + "à": "a", + "á": "a", + "â": "a", + "ã": "a", + "ä": "a", + "å": "a", + "æ": "ae", + "ç": "c", + "è": "e", + "é": "e", + "ê": "e", + "ë": "e", + "ì": "i", + "í": "i", + "î": "i", + "ï": "i", + "ð": "d", + "ñ": "n", + "ò": "o", + "ó": "o", + "ô": "o", + "õ": "o", + "ö": "o", + "ő": "o", + "ø": "o", + "ù": "u", + "ú": "u", + "û": "u", + "ü": "u", + "ű": "u", + "ý": "y", + "þ": "th", + "ÿ": "y", + "ẞ": "SS", + // greek + "α": "a", + "β": "b", + "γ": "g", + "δ": "d", + "ε": "e", + "ζ": "z", + "η": "h", + "θ": "8", + "ι": "i", + "κ": "k", + "λ": "l", + "μ": "m", + "ν": "n", + "ξ": "3", + "ο": "o", + "π": "p", + "ρ": "r", + "σ": "s", + "τ": "t", + "υ": "y", + "φ": "f", + "χ": "x", + "ψ": "ps", + "ω": "w", + "ά": "a", + "έ": "e", + "ί": "i", + "ό": "o", + "ύ": "y", + "ή": "h", + "ώ": "w", + "ς": "s", + "ϊ": "i", + "ΰ": "y", + "ϋ": "y", + "ΐ": "i", + "Α": "A", + "Β": "B", + "Γ": "G", + "Δ": "D", + "Ε": "E", + "Ζ": "Z", + "Η": "H", + "Θ": "8", + "Ι": "I", + "Κ": "K", + "Λ": "L", + "Μ": "M", + "Ν": "N", + "Ξ": "3", + "Ο": "O", + "Π": "P", + "Ρ": "R", + "Σ": "S", + "Τ": "T", + "Υ": "Y", + "Φ": "F", + "Χ": "X", + "Ψ": "PS", + "Ω": "W", + "Ά": "A", + "Έ": "E", + "Ί": "I", + "Ό": "O", + "Ύ": "Y", + "Ή": "H", + "Ώ": "W", + "Ϊ": "I", + "Ϋ": "Y", + // turkish + "ş": "s", + "Ş": "S", + "ı": "i", + "İ": "I", + "ğ": "g", + "Ğ": "G", + // russian + "а": "a", + "б": "b", + "в": "v", + "г": "g", + "д": "d", + "е": "e", + "ё": "yo", + "ж": "zh", + "з": "z", + "и": "i", + "й": "j", + "к": "k", + "л": "l", + "м": "m", + "н": "n", + "о": "o", + "п": "p", + "р": "r", + "с": "s", + "т": "t", + "у": "u", + "ф": "f", + "х": "h", + "ц": "c", + "ч": "ch", + "ш": "sh", + "щ": "sh", + "ъ": "u", + "ы": "y", + "ь": "", + "э": "e", + "ю": "yu", + "я": "ya", + "А": "A", + "Б": "B", + "В": "V", + "Г": "G", + "Д": "D", + "Е": "E", + "Ё": "Yo", + "Ж": "Zh", + "З": "Z", + "И": "I", + "Й": "J", + "К": "K", + "Л": "L", + "М": "M", + "Н": "N", + "О": "O", + "П": "P", + "Р": "R", + "С": "S", + "Т": "T", + "У": "U", + "Ф": "F", + "Х": "H", + "Ц": "C", + "Ч": "Ch", + "Ш": "Sh", + "Щ": "Sh", + "Ъ": "U", + "Ы": "Y", + "Ь": "", + "Э": "E", + "Ю": "Yu", + "Я": "Ya", + // ukranian + "Є": "Ye", + "І": "I", + "Ї": "Yi", + "Ґ": "G", + "є": "ye", + "і": "i", + "ї": "yi", + "ґ": "g", + // czech + "č": "c", + "ď": "d", + "ě": "e", + "ň": "n", + "ř": "r", + "š": "s", + "ť": "t", + "ů": "u", + "ž": "z", + "Č": "C", + "Ď": "D", + "Ě": "E", + "Ň": "N", + "Ř": "R", + "Š": "S", + "Ť": "T", + "Ů": "U", + "Ž": "Z", + // polish + "ą": "a", + "ć": "c", + "ę": "e", + "ł": "l", + "ń": "n", + "ś": "s", + "ź": "z", + "ż": "z", + "Ą": "A", + "Ć": "C", + "Ę": "e", + "Ł": "L", + "Ń": "N", + "Ś": "S", + "Ź": "Z", + "Ż": "Z", + // latvian + "ā": "a", + "ē": "e", + "ģ": "g", + "ī": "i", + "ķ": "k", + "ļ": "l", + "ņ": "n", + "ū": "u", + "Ā": "A", + "Ē": "E", + "Ģ": "G", + "Ī": "i", + "Ķ": "k", + "Ļ": "L", + "Ņ": "N", + "Ū": "u" +}; \ No newline at end of file diff --git a/src/constants/settings.ts b/src/constants/settings.ts index 4b49a18e..72be7432 100644 --- a/src/constants/settings.ts +++ b/src/constants/settings.ts @@ -4,4 +4,7 @@ export const CONFIG_KEY = "frontMatter"; export const ACTION_TAXONOMY_TAGS = "taxonomy.tags"; export const ACTION_TAXONOMY_CATEGORIES = "taxonomy.categories"; -export const ACTION_DATE_FORMAT = "taxonomy.dateFormat"; \ No newline at end of file +export const ACTION_DATE_FORMAT = "taxonomy.dateFormat"; + +export const ACTION_SLUG_PREFIX = "taxonomy.slugPrefix"; +export const ACTION_SLUG_SUFFIX = "taxonomy.slugSuffix"; \ No newline at end of file diff --git a/src/constants/stopwords-en.ts b/src/constants/stopwords-en.ts new file mode 100644 index 00000000..dbe154a0 --- /dev/null +++ b/src/constants/stopwords-en.ts @@ -0,0 +1,661 @@ +export const stopWords = [ + "a", + "able", + "about", + "above", + "abroad", + "according", + "accordingly", + "across", + "actually", + "adj", + "after", + "afterwards", + "again", + "against", + "ago", + "ahead", + "ain't", + "all", + "allow", + "allows", + "almost", + "alone", + "along", + "alongside", + "already", + "also", + "although", + "always", + "am", + "amid", + "amidst", + "among", + "amongst", + "an", + "and", + "another", + "any", + "anybody", + "anyhow", + "anyone", + "anything", + "anyway", + "anyways", + "anywhere", + "apart", + "appear", + "appreciate", + "appropriate", + "are", + "aren't", + "around", + "as", + "a's", + "aside", + "ask", + "asking", + "associated", + "at", + "available", + "away", + "awfully", + "b", + "back", + "backward", + "backwards", + "be", + "became", + "because", + "become", + "becomes", + "becoming", + "been", + "before", + "beforehand", + "begin", + "behind", + "being", + "believe", + "below", + "beside", + "besides", + "best", + "better", + "between", + "beyond", + "both", + "brief", + "but", + "by", + "c", + "came", + "can", + "cannot", + "cant", + "can't", + "caption", + "cause", + "causes", + "certain", + "certainly", + "changes", + "clearly", + "c'mon", + "co", + "co.", + "com", + "come", + "comes", + "concerning", + "consequently", + "consider", + "considering", + "contain", + "containing", + "contains", + "corresponding", + "could", + "couldn't", + "course", + "c's", + "currently", + "d", + "dare", + "daren't", + "definitely", + "described", + "despite", + "did", + "didn't", + "different", + "directly", + "do", + "does", + "doesn't", + "doing", + "done", + "don't", + "down", + "downwards", + "during", + "e", + "each", + "edu", + "eg", + "eight", + "eighty", + "either", + "else", + "elsewhere", + "end", + "ending", + "enough", + "entirely", + "especially", + "et", + "etc", + "even", + "ever", + "evermore", + "every", + "everybody", + "everyone", + "everything", + "everywhere", + "ex", + "exactly", + "example", + "except", + "f", + "fairly", + "far", + "farther", + "few", + "fewer", + "fifth", + "first", + "five", + "followed", + "following", + "follows", + "for", + "forever", + "former", + "formerly", + "forth", + "forward", + "found", + "four", + "from", + "further", + "furthermore", + "g", + "get", + "gets", + "getting", + "given", + "gives", + "go", + "goes", + "going", + "gone", + "got", + "gotten", + "greetings", + "h", + "had", + "hadn't", + "half", + "happens", + "hardly", + "has", + "hasn't", + "have", + "haven't", + "having", + "he", + "he'd", + "he'll", + "hello", + "help", + "hence", + "her", + "here", + "hereafter", + "hereby", + "herein", + "here's", + "hereupon", + "hers", + "herself", + "he's", + "hi", + "him", + "himself", + "his", + "hither", + "hopefully", + "how", + "howbeit", + "however", + "hundred", + "i", + "i'd", + "ie", + "if", + "ignored", + "i'll", + "i'm", + "immediate", + "in", + "inasmuch", + "inc", + "inc.", + "indeed", + "indicate", + "indicated", + "indicates", + "inner", + "inside", + "insofar", + "instead", + "into", + "inward", + "is", + "isn't", + "it", + "it'd", + "it'll", + "its", + "it's", + "itself", + "i've", + "j", + "just", + "k", + "keep", + "keeps", + "kept", + "know", + "known", + "knows", + "l", + "last", + "lately", + "later", + "latter", + "latterly", + "least", + "less", + "lest", + "let", + "let's", + "like", + "liked", + "likely", + "likewise", + "little", + "look", + "looking", + "looks", + "low", + "lower", + "ltd", + "m", + "made", + "mainly", + "make", + "makes", + "many", + "may", + "maybe", + "mayn't", + "me", + "mean", + "meantime", + "meanwhile", + "merely", + "might", + "mightn't", + "mine", + "minus", + "miss", + "more", + "moreover", + "most", + "mostly", + "mr", + "mrs", + "much", + "must", + "mustn't", + "my", + "myself", + "n", + "name", + "namely", + "nd", + "near", + "nearly", + "necessary", + "need", + "needn't", + "needs", + "neither", + "never", + "neverf", + "neverless", + "nevertheless", + "new", + "next", + "nine", + "ninety", + "no", + "nobody", + "non", + "none", + "nonetheless", + "noone", + "no-one", + "nor", + "normally", + "not", + "nothing", + "notwithstanding", + "novel", + "now", + "nowhere", + "o", + "obviously", + "of", + "off", + "often", + "oh", + "ok", + "okay", + "old", + "on", + "once", + "one", + "ones", + "one's", + "only", + "onto", + "opposite", + "or", + "other", + "others", + "otherwise", + "ought", + "oughtn't", + "our", + "ours", + "ourselves", + "out", + "outside", + "over", + "overall", + "own", + "p", + "particular", + "particularly", + "past", + "per", + "perhaps", + "placed", + "please", + "plus", + "possible", + "presumably", + "probably", + "provided", + "provides", + "q", + "que", + "quite", + "qv", + "r", + "rather", + "rd", + "re", + "really", + "reasonably", + "recent", + "recently", + "regarding", + "regardless", + "regards", + "relatively", + "respectively", + "right", + "round", + "s", + "said", + "same", + "saw", + "say", + "saying", + "says", + "second", + "secondly", + "see", + "seeing", + "seem", + "seemed", + "seeming", + "seems", + "seen", + "self", + "selves", + "sensible", + "sent", + "serious", + "seriously", + "seven", + "several", + "shall", + "shan't", + "she", + "she'd", + "she'll", + "she's", + "should", + "shouldn't", + "since", + "six", + "so", + "some", + "somebody", + "someday", + "somehow", + "someone", + "something", + "sometime", + "sometimes", + "somewhat", + "somewhere", + "soon", + "sorry", + "specified", + "specify", + "specifying", + "still", + "sub", + "such", + "sup", + "sure", + "t", + "take", + "taken", + "taking", + "tell", + "tends", + "th", + "than", + "thank", + "thanks", + "thanx", + "that", + "that'll", + "thats", + "that's", + "that've", + "the", + "their", + "theirs", + "them", + "themselves", + "then", + "thence", + "there", + "thereafter", + "thereby", + "there'd", + "therefore", + "therein", + "there'll", + "there're", + "theres", + "there's", + "thereupon", + "there've", + "these", + "they", + "they'd", + "they'll", + "they're", + "they've", + "thing", + "things", + "think", + "third", + "thirty", + "this", + "thorough", + "thoroughly", + "those", + "though", + "three", + "through", + "throughout", + "thru", + "thus", + "till", + "to", + "together", + "too", + "took", + "toward", + "towards", + "tried", + "tries", + "truly", + "try", + "trying", + "t's", + "twice", + "two", + "u", + "un", + "under", + "underneath", + "undoing", + "unfortunately", + "unless", + "unlike", + "unlikely", + "until", + "unto", + "up", + "upon", + "upwards", + "us", + "use", + "used", + "useful", + "uses", + "using", + "usually", + "v", + "value", + "various", + "versus", + "very", + "via", + "viz", + "vs", + "w", + "want", + "wants", + "was", + "wasn't", + "way", + "we", + "we'd", + "welcome", + "well", + "we'll", + "went", + "were", + "we're", + "weren't", + "we've", + "what", + "whatever", + "what'll", + "what's", + "what've", + "when", + "whence", + "whenever", + "where", + "whereafter", + "whereas", + "whereby", + "wherein", + "where's", + "whereupon", + "wherever", + "whether", + "which", + "whichever", + "while", + "whilst", + "whither", + "who", + "who'd", + "whoever", + "whole", + "who'll", + "whom", + "whomever", + "who's", + "whose", + "why", + "will", + "willing", + "wish", + "with", + "within", + "without", + "wonder", + "won't", + "would", + "wouldn't", + "x", + "y", + "yes", + "yet", + "you", + "you'd", + "you'll", + "your", + "you're", + "yours", + "yourself", + "yourselves", + "you've", + "z", + "zero" +]; \ No newline at end of file diff --git a/src/extension.ts b/src/extension.ts index 7b003096..ab7a7e62 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -35,6 +35,10 @@ export function activate({ subscriptions }: vscode.ExtensionContext) { Article.setDate(); }); + let generateSlug = vscode.commands.registerCommand('frontMatter.generateSlug', () => { + Article.generateSlug(); + }); + const toggleDraftCommand = 'frontMatter.toggleDraft'; const toggleDraft = vscode.commands.registerCommand(toggleDraftCommand, async () => { await Article.toggleDraft(); @@ -60,6 +64,7 @@ export function activate({ subscriptions }: vscode.ExtensionContext) { subscriptions.push(exportTaxonomy); subscriptions.push(remap); subscriptions.push(setDate); + subscriptions.push(generateSlug); subscriptions.push(toggleDraft); } diff --git a/src/helpers/ArticleHelper.ts b/src/helpers/ArticleHelper.ts index 6397994b..5dbc1b96 100644 --- a/src/helpers/ArticleHelper.ts +++ b/src/helpers/ArticleHelper.ts @@ -1,5 +1,7 @@ import * as vscode from 'vscode'; import * as matter from "gray-matter"; +import { stopWords } from '../constants/stopwords-en'; +import { charMap } from '../constants/charMap'; export class ArticleHelper { @@ -27,4 +29,62 @@ export class ArticleHelper { const nrOfLines = editor.document.lineCount as number; await editor.edit(builder => builder.replace(new vscode.Range(new vscode.Position(0, 0), new vscode.Position(nrOfLines, 0)), newMarkdown)); } + + /** + * Generate the slug + * + * @param articleTitle + */ + static createSlug(articleTitle: string): string | null { + if (!articleTitle) { + return null; + } + + // Remove punctuation from input string, and split it into words. + let cleanTitle = this.removePunctuation(articleTitle); + cleanTitle = cleanTitle.toLowerCase(); + // Split into words + let words = cleanTitle.split(/\s/); + // Removing stop words + words = this.removeStopWords(words); + cleanTitle = words.join("-"); + cleanTitle = this.replaceCharacters(cleanTitle); + return cleanTitle; + } + + /** + * Remove links, periods, commas, semi-colons, etc. + * + * @param value + */ + private static removePunctuation(value: string): string { + const punctuationless = value.replace(/[\.,-\/#!$@%\^&\*;:{}=\-_`'"~()+\?<>]/g, " "); + // Remove double spaces + return punctuationless.replace(/\s{2,}/g," "); + } + + /** + * Remove stop words + * + * @param words + */ + private static removeStopWords(words: string[]) { + const validWords: string[] = []; + for (const word of words) { + if (stopWords.indexOf(word.toLowerCase()) === -1) { + validWords.push(word); + } + } + return validWords; + } + + /** + * Replace characters from title + * + * @param value + */ + private static replaceCharacters(value: string) { + const characters = [...value]; + return characters.map(c => charMap[c] || c).join(""); + } } \ No newline at end of file