diff --git a/package.json b/package.json index b3fb9d11ccc7f0645c964712220c24e6b2c24036..666586bbdc158ff6300dc3e271f186dcd8fe84de 100644 --- a/package.json +++ b/package.json @@ -1,15 +1,16 @@ { "name": "frame-bot", "version": "2.0.0", - "description": "next-gen frame-botter", + "description": "download images and tweet them also make images/thumbnails from videos", "main": "src/index.js", "scripts": { "start": "node src/index.js", "ready": "node src/ready.js", - "analyze": "node src/analyze.js", - "make": "node src/make.js", + "analyze-videos": "node src/analyze.js", + "make-frames": "node src/make.js", "login": "node src/login.js", - "dropbox": "node src/dropbox.js", + "analyze-dropbox": "node src/dropbox.js", + "analyze-s3": "node src/s3.js", "tweet": "node src/tweet.js", "test": "mocha --require test/init.js --recursive --exit" }, @@ -29,6 +30,7 @@ }, "homepage": "https://gitlab.com/dudethatbe/frame-bot#readme", "dependencies": { + "aws-sdk": "^2.967.0", "cookie-parser": "^1.4.5", "dotenv": "^10.0.0", "dropbox": "^10.4.0", diff --git a/src/bin/analyze_s3.js b/src/bin/analyze_s3.js new file mode 100644 index 0000000000000000000000000000000000000000..6da7f717c26c0f5321e9e1fc13d01b8204f405f5 --- /dev/null +++ b/src/bin/analyze_s3.js @@ -0,0 +1,45 @@ +const { chooseRandomObject, downloadObject, getObjects } = require("../lib/s3"); +const { parse } = require("path"); +const { fileIsImage } = require("../lib/videoFinder"); +module.exports.downloadRandomFile = async (bucket, folder) => { + const object = await chooseRandomObject(bucket, folder); + const file = await downloadObject(bucket, object.Key); + return file; +}; +module.exports.analyzeFolder = async (bucket, folder) => { + if (!folder) { + folder = ""; + console.log(`Looking up objects in bucket ${bucket}`); + } else { + console.log(`Looking up files/folders uploaded to ${folder} ...`); + } + const label = `Time to scan ${folder}`; + console.time(label); + const results = await getObjects(bucket, folder); + console.timeEnd(label); + + let files = []; + const folders = new Set(); + let folderSize = 0; + for (const result of results) { + let size = result["Size"]; + const key = result["Key"]; + const p = parse(key); + const file = p.base; + const subFolder = p.dir; + if (fileIsImage(file)) { + files.push(file); + } + folders.add(subFolder); + folderSize += size; + } + + const numberOfFolders = folders.size; + const numberOfFiles = files.length; + const size = `${(folderSize * 1e-9).toFixed(2)}GB`; + + console.log(`Number of folders\t${numberOfFolders}`); + console.log(`Number of files\t\t${numberOfFiles}`); + console.log(`Estimated size\t\t${size}`); + return results; +}; diff --git a/src/bin/tweet.js b/src/bin/tweet.js new file mode 100644 index 0000000000000000000000000000000000000000..853c613c61a262a7e95c138131d1e246f7fddbf6 --- /dev/null +++ b/src/bin/tweet.js @@ -0,0 +1,36 @@ +const { unlink } = require("fs").promises; + +const { chooseRandomFrame, downloadFile } = require("../lib/dropbox"); +const { chooseRandomObject, downloadObject } = require("../lib/s3"); +const { postTweet } = require("../lib/twitter"); + +async function chooseFileFromDropbox(folder, saveTo) { + console.log(`Choosing a file to download from ${folder} ...`); + const randomFrame = await chooseRandomFrame(folder); + const path = randomFrame.path_lower; + const { file, response } = await downloadFile(path, saveTo); + console.log(`Downloaded ${file}`); + return file; +} +async function chooseFileFromSpaces(folder, bucket, saveTo) { + console.log(`Choosing a file to download from [${bucket}]/${folder} ...`); + const object = await chooseRandomObject(bucket, folder); + const file = await downloadObject(bucket, object["Key"], saveTo); + console.log(`Downloaded ${object["Key"]}`); // print out the filename aka the "Key" + return file; +} +async function tweetFile(file) { + const tweet = await postTweet(file); + console.log(`${tweet.created_at} => ${tweet.text}`); + return tweet; +} +exports.tweetFromDropbox = async (folder, saveToFolder) => { + const file = await chooseFileFromDropbox(folder, saveToFolder); + await tweetFile(file); + await unlink(file); +}; +exports.tweetFromSpaces = async (folder, bucket, saveToFolder) => { + const file = await chooseFileFromSpaces(folder, bucket, saveToFolder); + await tweetFile(file); + await unlink(file); +}; diff --git a/src/lib/s3.js b/src/lib/s3.js new file mode 100644 index 0000000000000000000000000000000000000000..c912fa3803a8103b4fa73619cef9342b798f0b26 --- /dev/null +++ b/src/lib/s3.js @@ -0,0 +1,60 @@ +const AWS = require("aws-sdk"); +const fs = require("fs"); +const path = require("path"); +const spacesEndPoint = new AWS.Endpoint(process.env.ENDPOINT); +const spacesRegion = process.env.REGION; +const s3 = new AWS.S3({ + endpoint: spacesEndPoint, + region: spacesRegion, +}); +async function listObjects(bucket, folder, token) { + var params = { + Bucket: bucket, + }; + if (token) { + params["ContinuationToken"] = token; + params["Prefix"] = ""; + } else { + params["Prefix"] = folder; + } + + try { + const promise = await s3.listObjectsV2(params).promise(); + return promise; + } catch (e) { + console.error(e); + } +} +exports.getObjects = async (bucket, folder) => { + let response = await listObjects(bucket, folder); + let contents = response.Contents; + let objects = []; + objects = objects.concat(contents); + while (response.IsTruncated) { + let token = response.NextContinuationToken; + response = await listObjects(bucket, folder, token); + contents = response.Contents; + objects = objects.concat(contents); + } + return objects; +}; +exports.chooseRandomObject = async (bucket, folder) => { + const objects = await this.getObjects(bucket, folder); + const rand = Math.floor(Math.random() * objects.length); + return objects[rand]; +}; +exports.downloadObject = async (bucket, key, dest) => { + const file = path.resolve(dest, path.basename(key)); + const write = fs.createWriteStream(file); + return new Promise((resolve, reject) => { + s3.getObject({ Bucket: bucket, Key: key }) + .createReadStream() + .on("end", () => { + resolve(path.resolve(file)); + }) + .on("error", (e) => { + reject(e); + }) + .pipe(write); + }); +}; \ No newline at end of file diff --git a/src/lib/videoFinder.js b/src/lib/videoFinder.js index 73fbbbbed1a081fcc762edc26b11b725c859895d..77def39b1eb87825906b9ab0128338231a8bab0b 100644 --- a/src/lib/videoFinder.js +++ b/src/lib/videoFinder.js @@ -9,6 +9,9 @@ const exec = promisify(require("child_process").exec); exports.fileIsVideo = fileName => { return /webm|mpeg|mkv|flv|ogv|avi|mp4|m4a$/.test(fileName); }; +exports.fileIsImage = (fileName) => { + return /png|jpg|gif/.test(fileName); +}; exports.getVideoFilesFromPath = async pathToFiles => { let dirents; try { diff --git a/src/s3.js b/src/s3.js new file mode 100644 index 0000000000000000000000000000000000000000..cba656b610ac4cc9d648d1a9d9262bda5c634fb2 --- /dev/null +++ b/src/s3.js @@ -0,0 +1,10 @@ +require("dotenv").config(); +const { analyzeFolder } = require("./bin/analyze_s3"); + +(async function (bucket, folder) { + bucket = bucket || process.env.BUCKET; + folder = folder || process.env.SPACES_FOLDER; + const result = await analyzeFolder(bucket, folder); + // console.log("result"); + // console.log(result); +})(process.argv[2], process.argv[3]); diff --git a/src/tweet.js b/src/tweet.js index 529aab58ac4bcfdb39f7b443455a4e9cb3cbc30c..4e25eb0df1542046bb205fb1293021ca8ab2ce36 100644 --- a/src/tweet.js +++ b/src/tweet.js @@ -1,25 +1,35 @@ require("dotenv").config(); -const { unlink } = require("fs").promises; -const { chooseRandomFrame, downloadFile } = require("./lib/dropbox"); -const { postTweet } = require("./lib/twitter"); -(async function (inputFolder, saveToFolder) { +(async function (inputFolder, frameSource, saveToFolder, frameBucket) { + if (inputFolder === "help") { + // just show the args and quit + console.log( + `npm run tweet folder(where to tweet from) source(spaces|dropbox|s3) save-to(destination for frame) bucket(only for spaces|s3)` + ); + return; + } const folder = inputFolder || process.env.FOLDER; if (!folder) { throw new Error( - "unable to detect FOLDER set in .env or through the command-line" + "unable to detect FOLDER in .env or through the command-line" ); } if (!saveToFolder) { // if no destination specified, then use "here" saveToFolder = "./"; } - console.log(`Choosing a file to download from ${folder} ...`); - const randomFrame = await chooseRandomFrame(folder); - const path = randomFrame.path_lower; - const { file, response } = await downloadFile(path, saveToFolder); - console.log(`Downloaded file to ${file}`); - const tweet = await postTweet(file); - console.log(`${tweet.created_at} => ${tweet.text}`); - await unlink(file); -})(process.argv[2], process.argv[3]); + + if (frameSource === "dropbox") { + const { tweetFromDropbox } = require("./bin/tweet"); + tweetFromDropbox(inputFolder, saveToFolder); + } else if (frameSource === "spaces") { + const { tweetFromSpaces } = require("./bin/tweet"); + const bucket = frameBucket || process.env.BUCKET; + if (!bucket) { + throw new Error("unable to detect BUCKET in .env"); + } + tweetFromSpaces(inputFolder, bucket, saveToFolder); + } else { + console.error(`Unrecognized frameSource "${frameSource}"`); + } +})(process.argv[2], process.argv[3], process.argv[4]);