From 152aa24f5442aa07e16a662068b38370f39d4856 Mon Sep 17 00:00:00 2001
From: Dakota Clark <dakotamusic@gmail.com>
Date: Fri, 13 Aug 2021 05:06:09 -0400
Subject: [PATCH 1/8] add aws support but actually DO spaces

---
 src/bin/analyze_s3.js | 15 +++++++++++++++
 src/lib/s3.js         | 25 +++++++++++++++++++++++++
 src/s3.js             |  9 +++++++++
 3 files changed, 49 insertions(+)
 create mode 100644 src/bin/analyze_s3.js
 create mode 100644 src/lib/s3.js
 create mode 100644 src/s3.js

diff --git a/src/bin/analyze_s3.js b/src/bin/analyze_s3.js
new file mode 100644
index 0000000..ac81bc2
--- /dev/null
+++ b/src/bin/analyze_s3.js
@@ -0,0 +1,15 @@
+const { listObjects } = require("../lib/s3");
+
+module.exports.listObjects = async (bucket, folder) => {
+  let response = await listObjects(bucket, folder);
+  let contents = response.Contents;
+  let objects = [];
+  objects = objects.concat(contents);
+  while (response.IsTruncated) {
+    let token = response.NextContinuationToken;
+    response = await listObjects(bucket, folder, token);
+    contents = response.Contents;
+    objects = objects.concat(contents);
+  }
+  return objects;
+};
diff --git a/src/lib/s3.js b/src/lib/s3.js
new file mode 100644
index 0000000..bc60382
--- /dev/null
+++ b/src/lib/s3.js
@@ -0,0 +1,25 @@
+const AWS = require("aws-sdk");
+const spacesEndPoint = new AWS.Endpoint(process.env.SPACES_ENDPOINT);
+
+const s3 = new AWS.S3({
+  endpoint: spacesEndPoint,
+  region: "nyc3",
+});
+exports.listObjects = async function (bucket, folder, token) {
+  var params = {
+    Prefix: folder,
+    Bucket: bucket,
+  };
+  if (token) {
+    // todo fix this
+    params["ContinuationToken"] = token;
+    params["Prefix"] = "";
+  }
+
+  try {
+    const promise = await s3.listObjectsV2(params).promise();
+    return promise;
+  } catch (e) {
+    console.error(e);
+  }
+};
diff --git a/src/s3.js b/src/s3.js
new file mode 100644
index 0000000..97d2bb4
--- /dev/null
+++ b/src/s3.js
@@ -0,0 +1,9 @@
+require("dotenv").config();
+const { listObjects } = require("./bin/analyze_s3");
+
+(async function (bucket, folder) {
+  bucket = bucket || process.env.BUCKET;
+  folder = folder || process.env.SPACES_FOLDER;
+  const result = await listObjects(bucket, folder);
+  console.log(result);
+})(process.argv[2], process.argv[3]);
-- 
GitLab


From e0ab4af4cfbe5b85f42e20b730cc1e9d3809f1a3 Mon Sep 17 00:00:00 2001
From: Dakota Clark <dakotamusic@gmail.com>
Date: Fri, 13 Aug 2021 05:20:49 -0400
Subject: [PATCH 2/8] select random file

---
 src/lib/s3.js | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/src/lib/s3.js b/src/lib/s3.js
index bc60382..976dae1 100644
--- a/src/lib/s3.js
+++ b/src/lib/s3.js
@@ -23,3 +23,21 @@ exports.listObjects = async function (bucket, folder, token) {
     console.error(e);
   }
 };
+exports.getObjects = async (bucket, folder) => {
+  let response = await this.listObjects(bucket, folder);
+  let contents = response.Contents;
+  let objects = [];
+  objects = objects.concat(contents);
+  while (response.IsTruncated) {
+    let token = response.NextContinuationToken;
+    response = await this.listObjects(bucket, folder, token);
+    contents = response.Contents;
+    objects = objects.concat(contents);
+  }
+  return objects;
+};
+exports.chooseRandomObject = async (bucket, folder) => {
+  const objects = await this.getObjects(bucket, folder);
+  const rand = Math.floor(Math.random() * objects.length);
+  return objects[rand];
+};
\ No newline at end of file
-- 
GitLab


From 702c64894014dca3b84b6db7fd8d0e7e831c6eaa Mon Sep 17 00:00:00 2001
From: Dakota Clark <dakotamusic@gmail.com>
Date: Fri, 13 Aug 2021 05:21:21 -0400
Subject: [PATCH 3/8] change bin to select

---
 src/bin/analyze_s3.js | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/src/bin/analyze_s3.js b/src/bin/analyze_s3.js
index ac81bc2..0b17ead 100644
--- a/src/bin/analyze_s3.js
+++ b/src/bin/analyze_s3.js
@@ -1,15 +1,7 @@
-const { listObjects } = require("../lib/s3");
+const { listObjects, chooseRandomObject } = require("../lib/s3");
 
 module.exports.listObjects = async (bucket, folder) => {
-  let response = await listObjects(bucket, folder);
-  let contents = response.Contents;
-  let objects = [];
-  objects = objects.concat(contents);
-  while (response.IsTruncated) {
-    let token = response.NextContinuationToken;
-    response = await listObjects(bucket, folder, token);
-    contents = response.Contents;
-    objects = objects.concat(contents);
-  }
-  return objects;
+  //actually get a random object for testing
+  const object = await chooseRandomObject(bucket, folder);
+  console.log(object);
 };
-- 
GitLab


From 0cf3815e4985e18d1131880a7d21be37964316f4 Mon Sep 17 00:00:00 2001
From: Dakota Clark <dakotamusic@gmail.com>
Date: Sun, 15 Aug 2021 02:37:05 -0400
Subject: [PATCH 4/8] add download function from s3

---
 package.json          |  2 ++
 src/bin/analyze_s3.js |  9 +++++++--
 src/lib/s3.js         | 18 ++++++++++++++++++
 3 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/package.json b/package.json
index b3fb9d1..70ce979 100644
--- a/package.json
+++ b/package.json
@@ -10,6 +10,7 @@
     "make": "node src/make.js",
     "login": "node src/login.js",
     "dropbox": "node src/dropbox.js",
+    "s3": "node src/s3.js",
     "tweet": "node src/tweet.js",
     "test": "mocha --require test/init.js --recursive --exit"
   },
@@ -29,6 +30,7 @@
   },
   "homepage": "https://gitlab.com/dudethatbe/frame-bot#readme",
   "dependencies": {
+    "aws-sdk": "^2.967.0",
     "cookie-parser": "^1.4.5",
     "dotenv": "^10.0.0",
     "dropbox": "^10.4.0",
diff --git a/src/bin/analyze_s3.js b/src/bin/analyze_s3.js
index 0b17ead..07894ef 100644
--- a/src/bin/analyze_s3.js
+++ b/src/bin/analyze_s3.js
@@ -1,7 +1,12 @@
-const { listObjects, chooseRandomObject } = require("../lib/s3");
+const {
+  listObjects,
+  chooseRandomObject,
+  downloadObject,
+} = require("../lib/s3");
 
 module.exports.listObjects = async (bucket, folder) => {
   //actually get a random object for testing
   const object = await chooseRandomObject(bucket, folder);
-  console.log(object);
+  const file = await downloadObject(bucket, object.Key);
+  console.log(file);
 };
diff --git a/src/lib/s3.js b/src/lib/s3.js
index 976dae1..3688670 100644
--- a/src/lib/s3.js
+++ b/src/lib/s3.js
@@ -1,4 +1,6 @@
 const AWS = require("aws-sdk");
+const fs = require("fs");
+const path = require("path");
 const spacesEndPoint = new AWS.Endpoint(process.env.SPACES_ENDPOINT);
 
 const s3 = new AWS.S3({
@@ -40,4 +42,20 @@ exports.chooseRandomObject = async (bucket, folder) => {
   const objects = await this.getObjects(bucket, folder);
   const rand = Math.floor(Math.random() * objects.length);
   return objects[rand];
+};
+exports.downloadObject = async (bucket, key) => {
+  const file = path.basename(key);
+  const write = fs.createWriteStream(file);
+  return new Promise((resolve, reject) => {
+    const stream = s3
+      .getObject({ Bucket: bucket, Key: key })
+      .createReadStream()
+      .on("end", () => {
+        resolve(path.resolve(file));
+      })
+      .on("error", (e) => {
+        reject(e);
+      })
+      .pipe(write);
+  });
 };
\ No newline at end of file
-- 
GitLab


From 70dd70aa755eee4db2c76ae8d2ecac50b1c36380 Mon Sep 17 00:00:00 2001
From: Dakota Clark <dakotamusic@gmail.com>
Date: Sun, 15 Aug 2021 05:34:37 -0400
Subject: [PATCH 5/8] analyze subfolders in bucket

---
 src/bin/analyze_s3.js  | 44 +++++++++++++++++++++++++++++++++---------
 src/lib/s3.js          |  8 ++++----
 src/lib/videoFinder.js |  3 +++
 src/s3.js              |  7 ++++---
 4 files changed, 46 insertions(+), 16 deletions(-)

diff --git a/src/bin/analyze_s3.js b/src/bin/analyze_s3.js
index 07894ef..c9c6201 100644
--- a/src/bin/analyze_s3.js
+++ b/src/bin/analyze_s3.js
@@ -1,12 +1,38 @@
-const {
-  listObjects,
-  chooseRandomObject,
-  downloadObject,
-} = require("../lib/s3");
-
-module.exports.listObjects = async (bucket, folder) => {
-  //actually get a random object for testing
+const { chooseRandomObject, downloadObject, getObjects } = require("../lib/s3");
+const { parse } = require("path");
+const { fileIsImage } = require("../lib/videoFinder");
+module.exports.downloadRandomFile = async (bucket, folder) => {
   const object = await chooseRandomObject(bucket, folder);
   const file = await downloadObject(bucket, object.Key);
-  console.log(file);
+  return file;
+};
+module.exports.analyzeFolder = async (bucket, folder) => {
+  console.log(`Looking up files/folders uploaded to ${folder} ...`);
+  const label = `Time to scan ${folder}`;
+  console.time(label);
+  const results = await getObjects(bucket, folder);
+  console.timeEnd(label);
+  let files = [];
+  const folders = new Set();
+  let folderSize = 0;
+  for (const result of results) {
+    let size = result["Size"];
+    const key = result["Key"];
+    const p = parse(key);
+    const file = p.base;
+    const subFolder = p.dir;
+    if (fileIsImage(file)) {
+      files.push(file);
+    }
+    folders.add(subFolder);
+    folderSize += size;
+  }
+  const numberOfFolders = folders.size;
+  const numberOfFiles = files.length;
+  const size = `${(folderSize * 1e-9).toFixed(2)}GB`;
+
+  console.log(`Number of folders\t${numberOfFolders}`);
+  console.log(`Number of files\t\t${numberOfFiles}`);
+  console.log(`Estimated size\t\t${size}`);
+  return results;
 };
diff --git a/src/lib/s3.js b/src/lib/s3.js
index 3688670..dbff192 100644
--- a/src/lib/s3.js
+++ b/src/lib/s3.js
@@ -7,7 +7,7 @@ const s3 = new AWS.S3({
   endpoint: spacesEndPoint,
   region: "nyc3",
 });
-exports.listObjects = async function (bucket, folder, token) {
+async function listObjects(bucket, folder, token) {
   var params = {
     Prefix: folder,
     Bucket: bucket,
@@ -24,15 +24,15 @@ exports.listObjects = async function (bucket, folder, token) {
   } catch (e) {
     console.error(e);
   }
-};
+}
 exports.getObjects = async (bucket, folder) => {
-  let response = await this.listObjects(bucket, folder);
+  let response = await listObjects(bucket, folder);
   let contents = response.Contents;
   let objects = [];
   objects = objects.concat(contents);
   while (response.IsTruncated) {
     let token = response.NextContinuationToken;
-    response = await this.listObjects(bucket, folder, token);
+    response = await listObjects(bucket, folder, token);
     contents = response.Contents;
     objects = objects.concat(contents);
   }
diff --git a/src/lib/videoFinder.js b/src/lib/videoFinder.js
index 73fbbbb..77def39 100644
--- a/src/lib/videoFinder.js
+++ b/src/lib/videoFinder.js
@@ -9,6 +9,9 @@ const exec = promisify(require("child_process").exec);
 exports.fileIsVideo = fileName => {
   return /webm|mpeg|mkv|flv|ogv|avi|mp4|m4a$/.test(fileName);
 };
+exports.fileIsImage = (fileName) => {
+  return /png|jpg|gif/.test(fileName);
+};
 exports.getVideoFilesFromPath = async pathToFiles => {
   let dirents;
   try {
diff --git a/src/s3.js b/src/s3.js
index 97d2bb4..cba656b 100644
--- a/src/s3.js
+++ b/src/s3.js
@@ -1,9 +1,10 @@
 require("dotenv").config();
-const { listObjects } = require("./bin/analyze_s3");
+const { analyzeFolder } = require("./bin/analyze_s3");
 
 (async function (bucket, folder) {
   bucket = bucket || process.env.BUCKET;
   folder = folder || process.env.SPACES_FOLDER;
-  const result = await listObjects(bucket, folder);
-  console.log(result);
+  const result = await analyzeFolder(bucket, folder);
+  // console.log("result");
+  // console.log(result);
 })(process.argv[2], process.argv[3]);
-- 
GitLab


From 8a832530668a4ff29c446ddef47620c79274bbab Mon Sep 17 00:00:00 2001
From: Dakota Clark <dakotamusic@gmail.com>
Date: Wed, 25 Aug 2021 02:32:56 -0400
Subject: [PATCH 6/8] refactor tweet for s3/spaces

---
 src/bin/analyze_s3.js |  9 ++++++++-
 src/bin/tweet.js      | 36 ++++++++++++++++++++++++++++++++++++
 src/lib/s3.js         | 15 +++++++--------
 src/tweet.js          | 38 ++++++++++++++++++++++++--------------
 4 files changed, 75 insertions(+), 23 deletions(-)
 create mode 100644 src/bin/tweet.js

diff --git a/src/bin/analyze_s3.js b/src/bin/analyze_s3.js
index c9c6201..6da7f71 100644
--- a/src/bin/analyze_s3.js
+++ b/src/bin/analyze_s3.js
@@ -7,11 +7,17 @@ module.exports.downloadRandomFile = async (bucket, folder) => {
   return file;
 };
 module.exports.analyzeFolder = async (bucket, folder) => {
-  console.log(`Looking up files/folders uploaded to ${folder} ...`);
+  if (!folder) {
+    folder = "";
+    console.log(`Looking up objects in bucket ${bucket}`);
+  } else {
+    console.log(`Looking up files/folders uploaded to ${folder} ...`);
+  }
   const label = `Time to scan ${folder}`;
   console.time(label);
   const results = await getObjects(bucket, folder);
   console.timeEnd(label);
+
   let files = [];
   const folders = new Set();
   let folderSize = 0;
@@ -27,6 +33,7 @@ module.exports.analyzeFolder = async (bucket, folder) => {
     folders.add(subFolder);
     folderSize += size;
   }
+  
   const numberOfFolders = folders.size;
   const numberOfFiles = files.length;
   const size = `${(folderSize * 1e-9).toFixed(2)}GB`;
diff --git a/src/bin/tweet.js b/src/bin/tweet.js
new file mode 100644
index 0000000..853c613
--- /dev/null
+++ b/src/bin/tweet.js
@@ -0,0 +1,36 @@
+const { unlink } = require("fs").promises;
+
+const { chooseRandomFrame, downloadFile } = require("../lib/dropbox");
+const { chooseRandomObject, downloadObject } = require("../lib/s3");
+const { postTweet } = require("../lib/twitter");
+
+async function chooseFileFromDropbox(folder, saveTo) {
+  console.log(`Choosing a file to download from ${folder} ...`);
+  const randomFrame = await chooseRandomFrame(folder);
+  const path = randomFrame.path_lower;
+  const { file, response } = await downloadFile(path, saveTo);
+  console.log(`Downloaded ${file}`);
+  return file;
+}
+async function chooseFileFromSpaces(folder, bucket, saveTo) {
+  console.log(`Choosing a file to download from [${bucket}]/${folder} ...`);
+  const object = await chooseRandomObject(bucket, folder);
+  const file = await downloadObject(bucket, object["Key"], saveTo);
+  console.log(`Downloaded ${object["Key"]}`); // print out the filename aka the "Key"
+  return file;
+}
+async function tweetFile(file) {
+  const tweet = await postTweet(file);
+  console.log(`${tweet.created_at} => ${tweet.text}`);
+  return tweet;
+}
+exports.tweetFromDropbox = async (folder, saveToFolder) => {
+  const file = await chooseFileFromDropbox(folder, saveToFolder);
+  await tweetFile(file);
+  await unlink(file);
+};
+exports.tweetFromSpaces = async (folder, bucket, saveToFolder) => {
+  const file = await chooseFileFromSpaces(folder, bucket, saveToFolder);
+  await tweetFile(file);
+  await unlink(file);
+};
diff --git a/src/lib/s3.js b/src/lib/s3.js
index dbff192..50b9697 100644
--- a/src/lib/s3.js
+++ b/src/lib/s3.js
@@ -1,21 +1,21 @@
 const AWS = require("aws-sdk");
 const fs = require("fs");
 const path = require("path");
-const spacesEndPoint = new AWS.Endpoint(process.env.SPACES_ENDPOINT);
-
+const spacesEndPoint = new AWS.Endpoint(process.env.ENDPOINT);
+const spacesRegion = process.env.REGION;
 const s3 = new AWS.S3({
   endpoint: spacesEndPoint,
   region: "nyc3",
 });
 async function listObjects(bucket, folder, token) {
   var params = {
-    Prefix: folder,
     Bucket: bucket,
   };
   if (token) {
-    // todo fix this
     params["ContinuationToken"] = token;
     params["Prefix"] = "";
+  } else {
+    params["Prefix"] = folder;
   }
 
   try {
@@ -43,12 +43,11 @@ exports.chooseRandomObject = async (bucket, folder) => {
   const rand = Math.floor(Math.random() * objects.length);
   return objects[rand];
 };
-exports.downloadObject = async (bucket, key) => {
-  const file = path.basename(key);
+exports.downloadObject = async (bucket, key, dest) => {
+  const file = path.resolve(dest, path.basename(key));
   const write = fs.createWriteStream(file);
   return new Promise((resolve, reject) => {
-    const stream = s3
-      .getObject({ Bucket: bucket, Key: key })
+    s3.getObject({ Bucket: bucket, Key: key })
       .createReadStream()
       .on("end", () => {
         resolve(path.resolve(file));
diff --git a/src/tweet.js b/src/tweet.js
index 529aab5..4e25eb0 100644
--- a/src/tweet.js
+++ b/src/tweet.js
@@ -1,25 +1,35 @@
 require("dotenv").config();
-const { unlink } = require("fs").promises;
 
-const { chooseRandomFrame, downloadFile } = require("./lib/dropbox");
-const { postTweet } = require("./lib/twitter");
-(async function (inputFolder, saveToFolder) {
+(async function (inputFolder, frameSource, saveToFolder, frameBucket) {
+  if (inputFolder === "help") {
+    // just show the args and quit
+    console.log(
+      `npm run tweet folder(where to tweet from) source(spaces|dropbox|s3) save-to(destination for frame) bucket(only for spaces|s3)`
+    );
+    return;
+  }
   const folder = inputFolder || process.env.FOLDER;
   if (!folder) {
     throw new Error(
-      "unable to detect FOLDER set in .env or through the command-line"
+      "unable to detect FOLDER in .env or through the command-line"
     );
   }
   if (!saveToFolder) {
     // if no destination specified, then use "here"
     saveToFolder = "./";
   }
-  console.log(`Choosing a file to download from ${folder} ...`);
-  const randomFrame = await chooseRandomFrame(folder);
-  const path = randomFrame.path_lower;
-  const { file, response } = await downloadFile(path, saveToFolder);
-  console.log(`Downloaded file to ${file}`);
-  const tweet = await postTweet(file);
-  console.log(`${tweet.created_at} => ${tweet.text}`);
-  await unlink(file);
-})(process.argv[2], process.argv[3]);
+
+  if (frameSource === "dropbox") {
+    const { tweetFromDropbox } = require("./bin/tweet");
+    tweetFromDropbox(inputFolder, saveToFolder);
+  } else if (frameSource === "spaces") {
+    const { tweetFromSpaces } = require("./bin/tweet");
+    const bucket = frameBucket || process.env.BUCKET;
+    if (!bucket) {
+      throw new Error("unable to detect BUCKET in .env");
+    }
+    tweetFromSpaces(inputFolder, bucket, saveToFolder);
+  } else {
+    console.error(`Unrecognized frameSource "${frameSource}"`);
+  }
+})(process.argv[2], process.argv[3], process.argv[4]);
-- 
GitLab


From 2b05e18e0bc2244933f0effa3fda8c6f29166ecb Mon Sep 17 00:00:00 2001
From: Dakota Clark <dakotamusic@gmail.com>
Date: Wed, 25 Aug 2021 02:35:34 -0400
Subject: [PATCH 7/8] update description and adjust script names

---
 package.json | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/package.json b/package.json
index 70ce979..666586b 100644
--- a/package.json
+++ b/package.json
@@ -1,16 +1,16 @@
 {
   "name": "frame-bot",
   "version": "2.0.0",
-  "description": "next-gen frame-botter",
+  "description": "download images and tweet them also make images/thumbnails from videos",
   "main": "src/index.js",
   "scripts": {
     "start": "node src/index.js",
     "ready": "node src/ready.js",
-    "analyze": "node src/analyze.js",
-    "make": "node src/make.js",
+    "analyze-videos": "node src/analyze.js",
+    "make-frames": "node src/make.js",
     "login": "node src/login.js",
-    "dropbox": "node src/dropbox.js",
-    "s3": "node src/s3.js",
+    "analyze-dropbox": "node src/dropbox.js",
+    "analyze-s3": "node src/s3.js",
     "tweet": "node src/tweet.js",
     "test": "mocha --require test/init.js --recursive --exit"
   },
-- 
GitLab


From 95005ca2a3dda56ebb7fe3be9eeaa52f560797fd Mon Sep 17 00:00:00 2001
From: Dakota Clark <dakotamusic@gmail.com>
Date: Wed, 25 Aug 2021 02:55:15 -0400
Subject: [PATCH 8/8] use region

---
 src/lib/s3.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib/s3.js b/src/lib/s3.js
index 50b9697..c912fa3 100644
--- a/src/lib/s3.js
+++ b/src/lib/s3.js
@@ -5,7 +5,7 @@ const spacesEndPoint = new AWS.Endpoint(process.env.ENDPOINT);
 const spacesRegion = process.env.REGION;
 const s3 = new AWS.S3({
   endpoint: spacesEndPoint,
-  region: "nyc3",
+  region: spacesRegion,
 });
 async function listObjects(bucket, folder, token) {
   var params = {
-- 
GitLab