From 18e359cc49f950cf6f60047275c7bcd0427f544b Mon Sep 17 00:00:00 2001 From: Karey Higuera Date: Sat, 6 Aug 2022 22:51:29 -0400 Subject: [PATCH 1/2] add option for condensed threads --- README.md | 7 ++++++ package.json | 2 -- src/main.ts | 53 ++++++++++++++++++++++++++++++--------- src/util.ts | 71 +++++++++++++++++++++++++++++++++------------------- 4 files changed, 93 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 5ed05a6..1cd26cb 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,13 @@ Nota bene: this will make a separate network request for each tweet. ttm -t ``` +#### Condensed threads +Instead of showing complete, individual tweets with profile picture, date, etc. when downloading a thread, this option will show the header once and then only show the tweet bodies, representing tweet threads as a cohesive body of text. A header will be shown if a different author appears in the thread, for example if you're downloading a conversation between various authors. + +```bash +ttm -T +``` + ### Custom File Name In order to save the tweet with a custom filename, pass the desired name to the `--filename` flag. You can use the variables `[[name]]`, `[[handle]]`, `[[text]]`, and `[[id]]` in your filename, which will be replaced according to the following chart. The file extension `.md` will also be added automatically. diff --git a/package.json b/package.json index 9090cb0..87675fe 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,6 @@ "build": "rollup --config rollup.config.js --environment BUILD:production" }, "dependencies": { - "@types/baretest": "^2.0.0", "array.prototype.flatmap": "^1.2.5", "axios": "^0.21.1", "axios-retry": "^3.1.8", @@ -47,7 +46,6 @@ "@types/node": "^14.14.37", "@typescript-eslint/eslint-plugin": "^5.4.0", "@typescript-eslint/parser": "^5.4.0", - "baretest": "^2.0.0", "eslint": "^7.32.0", "rollup": "^2.32.1", "tslib": "^2.2.0", diff --git a/src/main.ts b/src/main.ts index 2e966a1..72e32cb 100644 --- a/src/main.ts +++ b/src/main.ts @@ -80,7 +80,14 @@ const optionDefinitions: OptionDefinition[] = [ alias: 't', type: Boolean, description: - 'Save an entire tweet thread (starting with the final tweet) in a single document.', + 'Save an entire tweet thread in a single document. Use the link of the last tweet.', + }, + { + name: 'condensed_thread', + alias: 'T', + type: Boolean, + description: + 'Save an entire tweet thread in a single document, but only show the author on the first tweet or on author changes. Use the link of the last tweet.', }, ] @@ -140,29 +147,51 @@ if (!options.bearer) { const id = getTweetID(options) const main = async () => { - let tweet = await getTweet(id, options.bearer) - let final = '' + const tweets: Tweet[] = [] + let currentTweet: Tweet = await getTweet(id, options.bearer) + tweets.push(currentTweet) // special handling for threads - if (options.thread) { + if (options.thread || options.condensedThread) { // check if this is the head tweet - while (tweet.data.conversation_id !== tweet.data.id) { - const markdown = await buildMarkdown(tweet, options, 'thread') - final = markdown + final + while (currentTweet.data.conversation_id !== currentTweet.data.id) { // load in parent tweet - const [parent_tweet] = tweet.data.referenced_tweets.filter( + const [parent_tweet] = currentTweet.data.referenced_tweets.filter( (ref_tweet: ReferencedTweet) => ref_tweet.type === 'replied_to' ) - tweet = await getTweet(parent_tweet.id, options.bearer) + currentTweet = await getTweet(parent_tweet.id, options.bearer) + tweets.push(currentTweet) } } - const markdown = await buildMarkdown(tweet, options) - final = markdown + final + // reverse the thread so the tweets are in chronological order + tweets.reverse() + const markdowns = await Promise.all( + tweets.map(async (tweet, index) => { + return await buildMarkdown( + tweet, + options, + index === 0 ? 'normal' : 'thread', + index === 0 ? null : tweets[index - 1].includes.users[0] + ) + }) + ) + const firstTweet = tweets[0] + if (options.condensedThread) { + markdowns.push( + '', + '', + `[Thread link](https://twitter.com/${firstTweet.includes.users[0].username}/status/${firstTweet.data.id})` + ) + } + + const final = options.condensedThread + ? markdowns.join('\n\n') + : markdowns.join('\n\n---\n\n') if (options.clipboard) { copyToClipboard(final) } else { - writeTweet(tweet, final, options) + writeTweet(firstTweet, final, options) } } diff --git a/src/util.ts b/src/util.ts index 83b3b03..2949195 100644 --- a/src/util.ts +++ b/src/util.ts @@ -7,11 +7,11 @@ import fs from 'fs' import path from 'path' const fsp = fs.promises import chalk from 'chalk' -import {Media, Poll, Tweet} from './models' +import {Media, Poll, Tweet, User} from './models' import {CommandLineOptions} from 'command-line-args' import {URL, URLSearchParams} from 'url' import {unicodeSubstring} from './unicodeSubstring' -import { decode } from 'html-entities' +import {decode} from 'html-entities' axiosRetry(Axios, {retries: 3}) @@ -129,7 +129,7 @@ const getTweetFromTTM = async (id: string, bearer: string): Promise => { const ttmUrl = new URL(`https://ttm.kbravh.dev/api/tweet`) const params = new URLSearchParams({ tweet: id, - source: 'cli' + source: 'cli', }) return await Axios({ method: 'GET', @@ -338,8 +338,23 @@ export const testPath = async (path: string): Promise => export const buildMarkdown = async ( tweet: Tweet, options: CommandLineOptions, - type: 'normal' | 'thread' | 'quoted' = 'normal' + type: 'normal' | 'thread' | 'quoted' = 'normal', + previousAuthor?: User ): Promise => { + if (type === 'thread' && !previousAuthor) { + panic('A thread tweet must have a previous author') + } + + let text = decode(tweet.data.text) + const user = tweet.includes.users[0] + + const iscondensedThreadTweet = !( + type !== 'thread' || + (type === 'thread' && !options.condensedThread) + ) + + const showAuthor = (iscondensedThreadTweet && user.id !== previousAuthor.id) || !iscondensedThreadTweet + let metrics: string[] = [] if (options.metrics) { metrics = [ @@ -349,9 +364,6 @@ export const buildMarkdown = async ( ] } - let text = decode(tweet.data.text) - const user = tweet.includes.users[0] - /** * replace entities with markdown links */ @@ -406,19 +418,24 @@ export const buildMarkdown = async ( await downloadAssets(tweet, options) } - let markdown = [ - `![${user.username}](${ - options.assets - ? path.join( - getLocalAssetPath(options), - `${user.username}-${user.id}.jpg` - ) - : user.profile_image_url - })`, // profile image - `${user.name} ([@${user.username}](https://twitter.com/${user.username}))`, // name and handle - '\n', - `${text}`, // text of the tweet - ] + let markdown = [] + if (showAuthor) { + markdown.push( + `![${user.username}](${ + options.assets + ? path.join( + getLocalAssetPath(options), + `${user.username}-${user.id}.jpg` + ) + : user.profile_image_url + })`, // profile image + `${user.name} ([@${user.username}](https://twitter.com/${user.username}))`, // name and handle + '\n', + text + ) + } else { + markdown.push(text) + } // remove newlines from within tweet text to avoid breaking our formatting markdown = flatMap(markdown, line => line.split('\n')) @@ -450,17 +467,19 @@ export const buildMarkdown = async ( markdown = markdown.map(line => '> ' + line) } - // add original tweet link to end of tweet - markdown.push( - '\n\n' + - `[Tweet link](https://twitter.com/${user.username}/status/${tweet.data.id})` - ) + // add original tweet link to end of tweet if not a condensed thread + if (!options.condensedThread) { + markdown.push( + '\n\n' + + `[Tweet link](https://twitter.com/${user.username}/status/${tweet.data.id})` + ) + } switch (type) { case 'normal': return frontmatter.concat(markdown).join('\n') case 'thread': - return '\n\n---\n\n' + markdown.join('\n') + return markdown.join('\n') case 'quoted': return '\n\n' + markdown.join('\n') default: From a688b44d3fc1cd8a766014e09db6e0cb44825847 Mon Sep 17 00:00:00 2001 From: Karey Higuera Date: Sat, 6 Aug 2022 22:59:47 -0400 Subject: [PATCH 2/2] strip new lines from alt text --- README.md | 4 ++-- package.json | 2 +- src/util.ts | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1cd26cb..ac4380c 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ This command line tool allows you to quickly save a tweet in Markdown format. Th ⚠ **You'll need to have Node.js of at least `v10.x` to use this tool.** -You can install this CLI tool easily by running +You can install this CLI tool by running ```bash yarn global add tweet-to-markdown @@ -99,7 +99,7 @@ For Windows, have a look at [DOSKEY](https://superuser.com/a/560558). ### Copy to Clipboard -What if you want to just copy the Markdown to the clipboard instead of saving to a file? Why, it's as simple as just passing the `-c` (`--clipboard`) flag. +What if you want to just copy the Markdown to the clipboard instead of saving to a file? Just pass the `-c` (`--clipboard`) flag. ```bash ttm -c https://twitter.com/JoshWComeau/status/1213870628895428611 diff --git a/package.json b/package.json index 87675fe..ba38c50 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "tweet-to-markdown", - "version": "2.0.4", + "version": "2.1.0", "description": "Quickly save Tweets as Markdown files.", "main": "dist/main.js", "repository": "https://github.com/kbravh/tweet-to-markdown.git", diff --git a/src/util.ts b/src/util.ts index 2949195..a696ea1 100644 --- a/src/util.ts +++ b/src/util.ts @@ -305,12 +305,13 @@ export const createMediaElements = ( return media.map(medium => { switch (medium.type) { case 'photo': + const alt_text = medium.alt_text ? medium.alt_text.replace(/\n/g, ' ') : '' return options.assets - ? `\n![${medium.alt_text ?? medium.media_key}](${path.join( + ? `\n![${alt_text ?? medium.media_key}](${path.join( localAssetPath, `${medium.media_key}.jpg` )})` - : `\n![${medium.alt_text ?? medium.media_key}](${medium.url})` + : `\n![${alt_text ?? medium.media_key}](${medium.url})` default: break }