Use newleaf instead of YouTube API

README.MD

@@ -32,10 +32,6 @@ Run the server with `npm start`.
 
 If you want to make changes, run `npm run dev` to automatically reload the server and run tests whenever a file is saved.
 
-# Privacy Policy
-
-If you set the `youtubeAPIKey` option in `config.json`, you must follow [Google's Privacy Policy](https://policies.google.com/privacy) and [YouTube's Terms of Service](https://www.youtube.com/t/terms)
-
 # API Docs
 
 Available [here](https://github.com/ajayyy/SponsorBlock/wiki/API-Docs)

config.json.example

@@ -4,7 +4,7 @@
     "port": 80,
     "globalSalt": "[global salt (pepper) that is added to every ip before hashing to make it even harder for someone to decode the ip]",
     "adminUserID": "[the hashed id of the user who can perform admin actions]",
-    "youtubeAPIKey": null, //get this from Google Cloud Platform [optional]
+    "newLeafURL": "http://localhost:3241",
     "discordReportChannelWebhookURL": null, //URL from discord if you would like notifications when someone makes a report [optional]
     "discordFirstTimeSubmissionsWebhookURL": null, //URL from discord if you would like notifications when someone makes a first time submission [optional]
     "discordCompletelyIncorrectReportWebhookURL": null, //URL from discord if you would like notifications when someone reports a submission as completely incorrect [optional]

docker/docker-compose.yml

@@ -10,7 +10,7 @@ services:
       - ./database-export/:/opt/exports # To make this work, run chmod 777 ./database-exports
     ports:
       - 5432:5432
-    restart: always
+    restart: unless-stopped
   redis:
     container_name: redis
     image: redis
@@ -19,7 +19,15 @@ services:
       - ./redis/redis.conf:/usr/local/etc/redis/redis.conf
     ports:
       - 32773:6379
-    restart: always
+    restart: unless-stopped
+  newleaf:
+    image: abeltramo/newleaf:latest
+    container_name: newleaf
+    restart: unless-stopped
+    ports:
+      - 3241:3000
+    volumes:
+      - ./newleaf/configuration.py:/workdir/configuration.py
 
 volumes:
   database-data:

docker/newleaf/configuration.py

@@ -0,0 +1,17 @@
+# ==============================
+#  You MUST set these settings.
+# ==============================
+
+# A URL that this site can be accessed on. Do not include a trailing slash.
+website_origin = "http://newleaf:3000"
+
+
+# ==============================
+#  These settings are optional.
+# ==============================
+
+# The address of the interface to bind to.
+#bind_host = "0.0.0.0"
+
+# The port to bind to.
+#bind_port = 3000

package.json

@@ -23,8 +23,7 @@
     "pg": "^8.5.1",
     "redis": "^3.1.1",
     "sync-mysql": "^3.0.1",
-    "uuid": "^3.3.2",
+    "uuid": "^3.3.2"
-    "youtube-api": "^3.0.1"
   },
   "devDependencies": {
     "@types/better-sqlite3": "^5.4.0",

src/config.ts

@@ -44,7 +44,7 @@ addDefaults(config, {
         },
     },
     userCounterURL: null,
-    youtubeAPIKey: null,
+    newLeafURL: null,
     maxRewardTimePerSegmentInSeconds: 86400,
     postgres: null,
     dumpDatabase: {

src/routes/postSkipSegments.ts

@@ -1,7 +1,7 @@
 import {config} from '../config';
 import {Logger} from '../utils/logger';
 import {db, privateDB} from '../databases/databases';
-import {YouTubeAPI} from '../utils/youtubeApi';
+import {getMaxResThumbnail, YouTubeAPI} from '../utils/youtubeApi';
 import {getSubmissionUUID} from '../utils/getSubmissionUUID';
 import fetch from 'node-fetch';
 import isoDurations, { end } from 'iso8601-duration';
@@ -18,16 +18,11 @@ import { deleteLockCategories } from './deleteLockCategories';
 import { getCategoryActionType } from '../utils/categoryInfo';
 import { QueryCacher } from '../utils/queryCacher';
 import { getReputation } from '../utils/reputation';
+import { APIVideoData, APIVideoInfo } from '../types/youtubeApi.model';
 
-interface APIVideoInfo {
+async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: APIVideoData, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) {
-    err: string | boolean,
-    data?: any
-}
-
-async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: any, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) {
     const row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
     const userName = row !== undefined ? row.userName : null;
-    const video = youtubeData.items[0];
 
     let scopeName = "submissions.other";
     if (submissionCount <= 1) {
@@ -37,8 +32,8 @@ async function sendWebhookNotification(userID: string, videoID: string, UUID: st
     dispatchEvent(scopeName, {
         "video": {
             "id": videoID,
-            "title": video.snippet.title,
+            "title": youtubeData?.title,
-            "thumbnail": video.snippet.thumbnails.maxres ? video.snippet.thumbnails.maxres : null,
+            "thumbnail": getMaxResThumbnail(youtubeData) || null,
             "url": "https://www.youtube.com/watch?v=" + videoID,
         },
         "submission": {
@@ -76,7 +71,7 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID:
             method: 'POST',
             body: JSON.stringify({
                 "embeds": [{
-                    "title": data.items[0].snippet.title,
+                    "title": data?.title,
                     "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (parseInt(startTime.toFixed(0)) - 2),
                     "description": "Submission ID: " + UUID +
                         "\n\nTimestamp: " +
@@ -87,7 +82,7 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID:
                         "name": userID,
                     },
                     "thumbnail": {
-                        "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
+                        "url": getMaxResThumbnail(data) || "",
                     },
                 }],
             }),
@@ -177,10 +172,7 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
         const {err, data} = apiVideoInfo;
         if (err) return false;
 
-        // Check to see if video exists
+        const duration = apiVideoInfo?.data?.lengthSeconds;
-        if (data.pageInfo.totalResults === 0) return "No video exists with id " + submission.videoID;
-
-        const duration = getYouTubeVideoDuration(apiVideoInfo);
         const segments = submission.segments;
         let nbString = "";
         for (let i = 0; i < segments.length; i++) {
@@ -220,8 +212,7 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
             return a[0] - b[0] || a[1] - b[1];
         }));
 
-        let videoDuration = data.items[0].contentDetails.duration;
+        const videoDuration = data.lengthSeconds;
-        videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration));
         if (videoDuration != 0) {
             let allSegmentDuration = 0;
             //sum all segment times together
@@ -273,13 +264,8 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
     }
 }
 
-function getYouTubeVideoDuration(apiVideoInfo: APIVideoInfo): VideoDuration {
-    const duration = apiVideoInfo?.data?.items[0]?.contentDetails?.duration;
-    return duration ? isoDurations.toSeconds(isoDurations.parse(duration)) as VideoDuration : null;
-}
-
 async function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<APIVideoInfo> {
-    if (config.youtubeAPIKey !== null) {
+    if (config.newLeafURL !== null) {
         return YouTubeAPI.listVideos(videoID, ignoreCache);
     } else {
         return null;
@@ -375,7 +361,7 @@ export async function postSkipSegments(req: Request, res: Response) {
         // Don't use cache if we don't know the video duraton, or the client claims that it has changed
         apiVideoInfo = await getYouTubeVideoInfo(videoID, !videoDuration || videoDurationChanged(videoDuration));
     }
-    const apiVideoDuration = getYouTubeVideoDuration(apiVideoInfo);
+    const apiVideoDuration = apiVideoInfo?.data?.lengthSeconds as VideoDuration;
     if (!videoDuration || (apiVideoDuration && Math.abs(videoDuration - apiVideoDuration) > 2)) {
         // If api duration is far off, take that one instead (it is only precise to seconds, not millis)
         videoDuration = apiVideoDuration || 0 as VideoDuration;

src/routes/voteOnSponsorTime.ts

@@ -2,7 +2,7 @@ import {Request, Response} from 'express';
 import {Logger} from '../utils/logger';
 import {isUserVIP} from '../utils/isUserVIP';
 import fetch from 'node-fetch';
-import {YouTubeAPI} from '../utils/youtubeApi';
+import {getMaxResThumbnail, YouTubeAPI} from '../utils/youtubeApi';
 import {db, privateDB} from '../databases/databases';
 import {dispatchEvent, getVoteAuthor, getVoteAuthorRaw} from '../utils/webhookUtils';
 import {getFormattedTime} from '../utils/getFormattedTime';
@@ -57,11 +57,11 @@ async function sendWebhooks(voteData: VoteData) {
             webhookURL = config.discordCompletelyIncorrectReportWebhookURL;
         }
 
-        if (config.youtubeAPIKey !== null) {
+        if (config.newLeafURL !== null) {
             const { err, data } = await YouTubeAPI.listVideos(submissionInfoRow.videoID);
 
-            if (err || data.items.length === 0) {
+            if (err) {
-                if (err) Logger.error(err.toString());
+                Logger.error(err.toString());
                 return;
             }
             const isUpvote = voteData.incrementAmount > 0;
@@ -72,9 +72,9 @@ async function sendWebhooks(voteData: VoteData) {
                 },
                 "video": {
                     "id": submissionInfoRow.videoID,
-                    "title": data.items[0].snippet.title,
+                    "title": data?.title,
                     "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID,
-                    "thumbnail": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
+                    "thumbnail": getMaxResThumbnail(data) || null,
                 },
                 "submission": {
                     "UUID": voteData.UUID,
@@ -103,7 +103,7 @@ async function sendWebhooks(voteData: VoteData) {
                     method: 'POST',
                     body: JSON.stringify({
                         "embeds": [{
-                            "title": data.items[0].snippet.title,
+                            "title": data?.title,
                             "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID
                                 + "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2),
                             "description": "**" + voteData.row.votes + " Votes Prior | " +
@@ -120,7 +120,7 @@ async function sendWebhooks(voteData: VoteData) {
                                 "name": voteData.finalResponse?.finalMessage ?? getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission),
                             },
                             "thumbnail": {
-                                "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
+                                "url": getMaxResThumbnail(data) || "",
                             },
                         }],
                     }),

src/types/config.model.ts

@@ -6,7 +6,7 @@ export interface SBSConfig {
     mockPort?: number;
     globalSalt: string;
     adminUserID: string;
-    youtubeAPIKey?: string;
+    newLeafURL?: string;
     discordReportChannelWebhookURL?: string;
     discordFirstTimeSubmissionsWebhookURL?: string;
     discordCompletelyIncorrectReportWebhookURL?: string;

src/types/youtubeApi.model.ts

@@ -0,0 +1,111 @@
+export interface APIVideoData {
+    "title": string,
+    "videoId": string,
+    "videoThumbnails": [
+        {
+        "quality": string,
+        "url": string,
+        second__originalUrl: string,
+        "width": number,
+        "height": number
+        }
+    ],
+    
+    "description": string,
+    "descriptionHtml": string,
+    "published": number,
+    "publishedText": string,
+    
+    "keywords": string[],
+    "viewCount": number,
+    "likeCount": number,
+    "dislikeCount": number,
+    
+    "paid": boolean,
+    "premium": boolean,
+    "isFamilyFriendly": boolean,
+    "allowedRegions": string[],
+    "genre": string,
+    "genreUrl": string,
+    
+    "author": string,
+    "authorId": string,
+    "authorUrl": string,
+    "authorThumbnails": [
+        {
+        "url": string,
+        "width": number,
+        "height": number
+        }
+    ],
+    
+    "subCountText": string,
+    "lengthSeconds": number,
+    "allowRatings": boolean,
+    "rating": number,
+    "isListed": boolean,
+    "liveNow": boolean,
+    "isUpcoming": boolean,
+    "premiereTimestamp"?: number,
+    
+    "hlsUrl"?: string,
+    "adaptiveFormats": [
+        {
+        "index": string,
+        "bitrate": string,
+        "init": string,
+        "url": string,
+        "itag": string,
+        "type": string,
+        "clen": string,
+        "lmt": string,
+        "projectionType": number,
+        "container": string,
+        "encoding": string,
+        "qualityLabel"?: string,
+        "resolution"?: string
+        }
+    ],
+    "formatStreams": [
+        {
+        "url": string,
+        "itag": string,
+        "type": string,
+        "quality": string,
+        "container": string,
+        "encoding": string,
+        "qualityLabel": string,
+        "resolution": string,
+        "size": string
+        }
+    ],
+    "captions": [
+        {
+        "label": string,
+        "languageCode": string,
+        "url": string
+        }
+    ],
+    "recommendedVideos": [
+        {
+        "videoId": string,
+        "title": string,
+        "videoThumbnails": [
+            {
+            "quality": string,
+            "url": string,
+            "width": number,
+            "height": number
+            }
+        ],
+        "author": string,
+        "lengthSeconds": number,
+        "viewCountText": string
+        }
+    ]
+}
+
+export interface APIVideoInfo {
+    err: string | boolean,
+    data?: APIVideoData
+}

src/utils/youtubeApi.ts

@@ -1,22 +1,16 @@
+import fetch from 'node-fetch';
 import {config} from '../config';
 import {Logger} from './logger';
 import redis from './redis';
-// @ts-ignore
+import { APIVideoData, APIVideoInfo } from '../types/youtubeApi.model';
-import _youTubeAPI from 'youtube-api';
-
-_youTubeAPI.authenticate({
-    type: "key",
-    key: config.youtubeAPIKey,
-});
 
 export class YouTubeAPI {
-    static async listVideos(videoID: string, ignoreCache = false): Promise<{err: string | boolean, data?: any}> {
+    static async listVideos(videoID: string, ignoreCache = false): Promise<APIVideoInfo> {
-        const part = 'contentDetails,snippet';
         if (!videoID || videoID.length !== 11 || videoID.includes(".")) {
             return { err: "Invalid video ID" };
         }
 
-        const redisKey = "youtube.video." + videoID;
+        const redisKey = "yt.newleaf.video." + videoID;
         if (!ignoreCache) {
             const {err, reply} =  await redis.getAsync(redisKey);
 
@@ -25,34 +19,37 @@ export class YouTubeAPI {
 
                 return { err: err?.message, data: JSON.parse(reply) }
             }
-       }
+        }
+
+        if (!config.newLeafURL) return {err: "NewLeaf URL not found", data: null};
 
         try {
-            const { ytErr, data } = await new Promise((resolve) => _youTubeAPI.videos.list({
+            const result = await fetch(config.newLeafURL + "/api/v1/videos/" + videoID, { method: "GET" });
-                part,
-                id: videoID,
-            }, (ytErr: boolean | string, result: any) => resolve({ytErr, data: result?.data})));
 
-            if (!ytErr) {
+            if (result.ok) {
-                // Only set cache if data returned
+                const data = await result.json();
-                if (data.items.length > 0) {
+                if (data.error) {
-                    const { err: setErr } = await redis.setAsync(redisKey, JSON.stringify(data));
+                    return { err: data.err, data: null };
+                }
 
-                    if (setErr) {
+                redis.setAsync(redisKey, JSON.stringify(data)).then((result) => {
-                        Logger.warn(setErr.message);
+                    if (result?.err) {
+                        Logger.warn(result?.err.message);
                     } else {
                         Logger.debug("redis: video information cache set for: " + videoID);
                     }
-
+                });
-                    return { err: false, data }; // don't fail
+                
-                } else {
+                return { err: false, data };
-                    return { err: false, data }; // don't fail
-                }
             } else {
-                return { err: ytErr, data };
+                return { err: result.statusText, data: null };
             }
         } catch (err) {
             return {err, data: null}
         }       
     }
 }
+
+export function getMaxResThumbnail(apiInfo: APIVideoData): string | void {
+    return apiInfo?.videoThumbnails?.find((elem) => elem.quality === "maxres")?.second__originalUrl;
+}

test.json

@@ -3,7 +3,7 @@
     "mockPort": 8081,
     "globalSalt": "testSalt",
     "adminUserID": "testUserId",
-    "youtubeAPIKey": "",
+    "newLeafURL": "placeholder",
     "discordReportChannelWebhookURL": "http://127.0.0.1:8081/ReportChannelWebhook",
     "discordFirstTimeSubmissionsWebhookURL": "http://127.0.0.1:8081/FirstTimeSubmissionsWebhook",
     "discordCompletelyIncorrectReportWebhookURL": "http://127.0.0.1:8081/CompletelyIncorrectReportWebhook",

test/cases/postSkipSegments.ts

@@ -118,7 +118,7 @@ describe('postSkipSegments', () => {
         .then(async res => {
             if (res.status === 200) {
                 const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXZX"]);
-                if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 5010) {
+                if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 4980) {
                     done();
                 } else {
                     done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
@@ -140,7 +140,7 @@ describe('postSkipSegments', () => {
             body: JSON.stringify({
                 userID: "test",
                 videoID: "dQw4w9WgXZH",
-                videoDuration: 5010.20,
+                videoDuration: 4980.20,
                 segments: [{
                     segment: [1, 10],
                     category: "sponsor",
@@ -150,7 +150,7 @@ describe('postSkipSegments', () => {
         .then(async res => {
             if (res.status === 200) {
                 const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXZH"]);
-                if (row.startTime === 1 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 5010.20) {
+                if (row.startTime === 1 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 4980.20) {
                     done();
                 } else {
                     done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
@@ -237,6 +237,21 @@ describe('postSkipSegments', () => {
         }
     });
 
+    it('Should still not be allowed if youtube thinks duration is 0', (done: Done) => {
+        fetch(getbaseURL()
+            + "/api/postVideoSponsorTimes?videoID=noDuration&startTime=30&endTime=10000&userID=testing", {
+                method: 'POST',
+        })
+        .then(async res => {
+            if (res.status === 403) done(); // pass
+            else {
+                const body = await res.text();
+                done("non 403 status code: " + res.status + " (" + body + ")");
+            }
+        })
+        .catch(err => done("Couldn't call endpoint"));
+    });
+
     it('Should be able to submit a single time under a different service (JSON method)', (done: Done) => {
         fetch(getbaseURL()
             + "/api/postVideoSponsorTimes", {
@@ -666,34 +681,6 @@ describe('postSkipSegments', () => {
         .catch(err => done(err));
     });
 
-    it('Should be allowed if youtube thinks duration is 0', (done: Done) => {
-        fetch(getbaseURL()
-            + "/api/postVideoSponsorTimes?videoID=noDuration&startTime=30&endTime=10000&userID=testing", {
-                method: 'POST',
-        })
-        .then(async res => {
-            if (res.status === 200) done(); // pass
-            else {
-                const body = await res.text();
-                done("non 200 status code: " + res.status + " (" + body + ")");
-            }
-        })
-        .catch(err => done("Couldn't call endpoint"));
-    });
-
-    it('Should be rejected if not a valid videoID', (done: Done) => {
-        fetch(getbaseURL()
-            + "/api/postVideoSponsorTimes?videoID=knownWrongID&startTime=30&endTime=1000000&userID=testing")
-        .then(async res => {
-            if (res.status === 403) done(); // pass
-            else {
-                const body = await res.text();
-                done("non 403 status code: " + res.status + " (" + body + ")");
-            }
-        })
-        .catch(err => done("Couldn't call endpoint"));
-    });
-
     it('Should return 400 for missing params (Params method)', (done: Done) => {
         fetch(getbaseURL()
             + "/api/postVideoSponsorTimes?startTime=9&endTime=10&userID=test", {

test/youtubeMock.ts

@@ -1,28 +1,14 @@
-/*
+import { APIVideoData, APIVideoInfo } from "../src/types/youtubeApi.model";
-YouTubeAPI.videos.list({
-  part: "snippet",
-  id: videoID
-}, function (err, data) {});
- */
-
-// https://developers.google.com/youtube/v3/docs/videos
-
 
 export class YouTubeApiMock {
-    static async listVideos(videoID: string, ignoreCache = false): Promise<{err: string | boolean, data?: any}> {
+    static async listVideos(videoID: string, ignoreCache = false): Promise<APIVideoInfo> {
         const obj = {
             id: videoID
         };
 
         if (obj.id === "knownWrongID") {
             return {
-                err: null,
+                err: "No video found"
-                data: {
-                    pageInfo: {
-                        totalResults: 0,
-                    },
-                    items: [],
-                }
             };
         }
 
@@ -30,49 +16,35 @@ export class YouTubeApiMock {
             return {
                 err: null,
                 data: {
-                    pageInfo: {
+                    title: "Example Title",
-                        totalResults: 1,
+                    lengthSeconds: 0,
-                    },
+                    videoThumbnails: [
-                    items: [
                         {
-                            contentDetails: {
+                           quality: "maxres",
-                                duration: "PT0S",
+                           url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png",
-                            },
+                           second__originalUrl:"https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png",
-                            snippet: {
+                           width: 1280,
-                                title: "Example Title",
+                           height: 720
-                                thumbnails: {
-                                    maxres: {
-                                        url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png",
-                                    },
-                                },
-                            },
                         },
-                    ],
+                    ]
-                }
+                } as APIVideoData
             };
         } else {
             return {
                 err: null,
                 data: {
-                    pageInfo: {
+                    title: "Example Title",
-                        totalResults: 1,
+                    lengthSeconds: 4980,
-                    },
+                    videoThumbnails: [
-                    items: [
                         {
-                            contentDetails: {
+                           quality: "maxres",
-                                duration: "PT1H23M30S",
+                           url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png",
-                            },
+                           second__originalUrl:"https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png",
-                            snippet: {
+                           width: 1280,
-                                title: "Example Title",
+                           height: 720
-                                thumbnails: {
-                                    maxres: {
-                                        url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png",
-                                    },
-                                },
-                            },
                         },
-                    ],
+                    ]
-                }
+                } as APIVideoData
             };
         }
     }