diff --git a/.gitignore b/.gitignore index 29b1331..d040916 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,8 @@ bower_components # Compiled binary addons (https://nodejs.org/api/addons.html) build/Release +videos/ + # Dependency directories node_modules/ jspm_packages/ @@ -83,4 +85,6 @@ videodata/ *.DS_Store -*.mp3 \ No newline at end of file +*.mp3 + +*.jpg diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..2426d8b --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,17 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Launch Program", + "skipFiles": [ + "/**" + ], + "program": "${workspaceFolder}\\bin\\www" + } + ] +} \ No newline at end of file diff --git a/MIrogue.json b/MIrogue.json new file mode 100644 index 0000000..49dbd7c --- /dev/null +++ b/MIrogue.json @@ -0,0 +1 @@ +{"code":0,"msg":"","status":"success","result":{"text":"","sentence":[{"sentence":" Can you open the door?","confidence":"0.8300","startTime":1.22,"endTime":2.68},{"sentence":" Ethan, who are you? I buy the plane. Betty, can you open the door? Uh Can I have the door? Uh Maybe","confidence":"0.7800","startTime":3.24,"endTime":10.3},{"sentence":" open the door. When I tell you","confidence":"0.8700","startTime":10.3,"endTime":12.04},{"sentence":" on the plane over the door, and you get in the plane, not in the plane. I'm on the plane,","confidence":"0.7800","startTime":27.25,"endTime":32.72},{"sentence":" open the door.","confidence":"0.7500","startTime":32.72,"endTime":34.06},{"sentence":" Yeah,","confidence":"0.0000","startTime":35.29,"endTime":36.3},{"sentence":" yeah.","confidence":"0.0000","startTime":36.87,"endTime":37.9},{"sentence":" Right. Jim opens the door. Yeah Yeah Yeah Yeah, okay okay,","confidence":"0.6100","startTime":39.03,"endTime":43.6},{"sentence":" Ben's you open that door right now. I'm trying","confidence":"0.8300","startTime":48.35,"endTime":51.1},{"sentence":" come on bench and you open that door.","confidence":"0.8800","startTime":65.76,"endTime":68.14},{"sentence":" Come On Come On Come on,","confidence":"0.8300","startTime":68.48,"endTime":70.68},{"sentence":" come on.","confidence":"0.6700","startTime":76.42,"endTime":77.2},{"sentence":" Yes, honey.","confidence":"0.4200","startTime":77.36,"endTime":78.62},{"sentence":" Got bored!","confidence":"0.6000","startTime":83.95,"endTime":85},{"sentence":" Ben. Je knap that door the other door the other door? All right. Yes The other door? Okay Yes Yes Yes, sorry sorry, my bad, my back","confidence":"0.6900","startTime":85.36,"endTime":92.19},{"sentence":" noise","confidence":"0.9900","startTime":92.91,"endTime":94.92},{"sentence":" to","confidence":"0.5100","startTime":95.02,"endTime":98.01},{"sentence":" hello hello.","confidence":"0.6700","startTime":98.09,"endTime":99.96},{"sentence":" All right. Look and get out the parallel.","confidence":"0.7100","startTime":99.96,"endTime":102.08},{"sentence":" How's that?","confidence":"0.6700","startTime":102.08,"endTime":103.46}],"word":[]}} \ No newline at end of file diff --git a/Mivault.json b/Mivault.json new file mode 100644 index 0000000..72e9527 --- /dev/null +++ b/Mivault.json @@ -0,0 +1 @@ +{"code":0,"msg":"","status":"success","result":{"text":"","sentence":[{"sentence":" Is that before","confidence":"0.9500","startTime":0.47,"endTime":3.08},{"sentence":" yeah,","confidence":"0.2400","startTime":3.14,"endTime":4.76},{"sentence":" oh","confidence":"0.0000","startTime":5.01,"endTime":6.2},{"sentence":" for you.","confidence":"0.6400","startTime":16.52,"endTime":17.84},{"sentence":" One red,","confidence":"0.6100","startTime":21.08,"endTime":22.16},{"sentence":" yeah","confidence":"0.7000","startTime":22.6,"endTime":23.3},{"sentence":" yeah","confidence":"0.5600","startTime":24.2,"endTime":25.96},{"sentence":" noise","confidence":"0.9700","startTime":26.54,"endTime":27.48},{"sentence":" yeah.","confidence":"0.0000","startTime":27.8,"endTime":28.68},{"sentence":" uh","confidence":"0.5700","startTime":28.96,"endTime":30.2},{"sentence":" keep moving","confidence":"1.0000","startTime":33.51,"endTime":34.74},{"sentence":" yeah","confidence":"0.4700","startTime":35.08,"endTime":36.42},{"sentence":" post","confidence":"0.7100","startTime":36.42,"endTime":37.96},{"sentence":" toast","confidence":"1.0000","startTime":38.88,"endTime":39.88},{"sentence":" messy","confidence":"1.0000","startTime":43.88,"endTime":44.94},{"sentence":" noise.","confidence":"0.4900","startTime":45.2,"endTime":47.16},{"sentence":" uh","confidence":"0.5900","startTime":47.94,"endTime":49.04},{"sentence":" noise","confidence":"1.0000","startTime":66.93,"endTime":67.84},{"sentence":" noise","confidence":"0.9700","startTime":71.53,"endTime":73.68},{"sentence":" noise","confidence":"0.9600","startTime":76.65,"endTime":79.04},{"sentence":" noise","confidence":"0.9900","startTime":79.36,"endTime":81.08},{"sentence":" yeah","confidence":"0.7600","startTime":82.05,"endTime":84.96},{"sentence":" noise","confidence":"0.9900","startTime":85.76,"endTime":87.48},{"sentence":" yeah.","confidence":"0.2400","startTime":87.48,"endTime":90.98},{"sentence":" uh","confidence":"0.6500","startTime":90.98,"endTime":92.52},{"sentence":" noise","confidence":"0.9900","startTime":92.67,"endTime":94.04},{"sentence":" noise","confidence":"0.9300","startTime":94.04,"endTime":95.32},{"sentence":" noise","confidence":"1.0000","startTime":96.37,"endTime":97.28},{"sentence":" yeah","confidence":"0.6300","startTime":97.81,"endTime":100.28},{"sentence":" sure.","confidence":"0.4800","startTime":104.03,"endTime":104.96}],"word":[]}} \ No newline at end of file diff --git a/README.Md b/README.Md new file mode 100644 index 0000000..dadcb0f --- /dev/null +++ b/README.Md @@ -0,0 +1,12 @@ +# RUN INSTRUCTIONS +- npm start +- app runs on port 5001 + +# Dependent services +- Azure Vision API +- IBM Max scene classifier + - To run image: docker run -it -p 5000:5000 max-scene-classifier + - Runs on port 5000 + - Instructions to build and run availaible on https://github.com/IBM/MAX-Scene-Classifier +- Question-answer generation model + - running on port 5003 diff --git a/app.js b/app.js index 7541875..de4b212 100644 --- a/app.js +++ b/app.js @@ -5,9 +5,10 @@ var cookieParser = require('cookie-parser'); var logger = require('morgan'); var indexRouter = require('./routes/index'); const app = express(); +var cors = require('cors') - +app.use(cors()) // view engine setup app.set('views', path.join(__dirname, 'views')); app.set('view engine', 'ejs'); @@ -21,7 +22,7 @@ app.use('/', indexRouter); // catch 404 and forward to error handler app.use(function(req, res, next) { - next(createError(404)); + next(createError(404)); }); // error handler @@ -36,3 +37,4 @@ app.use(function(err, req, res, next) { }); module.exports = app; +// \ No newline at end of file diff --git a/audio_ocean8.json b/audio_ocean8.json new file mode 100644 index 0000000..e9bdbea --- /dev/null +++ b/audio_ocean8.json @@ -0,0 +1 @@ +{"code":0,"msg":"","status":"success","result":{"text":"","sentence":[{"sentence":" It's kind,","confidence":"0.3900","startTime":6.25,"endTime":7.28},{"sentence":" it's further on and on","confidence":"0.7700","startTime":7.54,"endTime":11.2},{"sentence":" Spain,","confidence":"0.2700","startTime":11.35,"endTime":12.16},{"sentence":" this country","confidence":"0.1500","startTime":12.28,"endTime":13.32},{"sentence":" just like me","confidence":"0.3600","startTime":13.44,"endTime":15.72},{"sentence":" noise,","confidence":"0.1600","startTime":15.72,"endTime":16.54},{"sentence":" yes","confidence":"0.9200","startTime":16.63,"endTime":20.4},{"sentence":" noise","confidence":"0.8100","startTime":20.4,"endTime":21.55},{"sentence":" noise","confidence":"0.9700","startTime":21.55,"endTime":23.12},{"sentence":" noise","confidence":"0.0100","startTime":23.3,"endTime":25.4},{"sentence":" noise","confidence":"0.8900","startTime":25.4,"endTime":26.16},{"sentence":" noise","confidence":"0.9400","startTime":26.16,"endTime":26.92},{"sentence":" noise.","confidence":"0.4500","startTime":27.15,"endTime":28.04},{"sentence":" Hi, I like to return these. Oh Of Course Do have you ever seen? No, but they're uh nope. And they haven't touched. I really need to receipt.","confidence":"0.7200","startTime":28.35,"endTime":36.57},{"sentence":" They are sealed, their brand new. Do you have the credit card that you use? This is ridiculous. I bought these last week. Maybe you can try client services on the 6th floor.","confidence":"0.8200","startTime":36.57,"endTime":45.85},{"sentence":" Never mind. Just keep them. Jesus","confidence":"0.7500","startTime":45.89,"endTime":48.96},{"sentence":" can at least get a bag. Sure.","confidence":"0.7800","startTime":49.27,"endTime":51.64},{"sentence":" Thank you.","confidence":"0.6700","startTime":54.81,"endTime":55.69},{"sentence":" I'm checking I mean 2084, very randal job less and stay graphic.","confidence":"0.6600","startTime":66.53,"endTime":71.49},{"sentence":" Okay, my name is monica. There anything else I can do for you? Or arrange transportation? The girl said,","confidence":"0.7800","startTime":72.81,"endTime":78.04},{"sentence":" hi, this is this is randall. We just checked out of room 2084. Sp. Quick! Monica, please. Is she? Hi, monica! Hi Hi. Uh We just found out our flight was cancelled. Know, I know.","confidence":"0.6700","startTime":79,"endTime":91.72},{"sentence":" No. And instead of saying some horrible airport hotel, we were hoping that maybe we could get our original room back. Yeah We can give you that same room back.","confidence":"0.8800","startTime":91.72,"endTime":99.96},{"sentence":" That would be amazing. Thank you so much. Um We're just gonna go grab a bite. Uh Would it be possible to get the maid in there now?","confidence":"0.8600","startTime":99.96,"endTime":107.22},{"sentence":" Yes, absolutely","confidence":"0.6700","startTime":107.22,"endTime":108.66},{"sentence":" perfect. We really appreciate it. It's not a problem. Thank you so much. We'll see you soon.","confidence":"0.7700","startTime":108.66,"endTime":113.52},{"sentence":" Hi,","confidence":"0.5000","startTime":124.92,"endTime":125.96},{"sentence":" I'm so sorry. Can you finish up later? I just have to get off my feet. Of Course Thank you so much. Have a great day. Who! Bye? Bye.","confidence":"0.7400","startTime":125.96,"endTime":132.75}],"word":[]}} \ No newline at end of file diff --git a/backend/audioDesc.js b/backend/audioDesc.js new file mode 100644 index 0000000..172c670 --- /dev/null +++ b/backend/audioDesc.js @@ -0,0 +1,1035 @@ +const textToSpeech = require("@google-cloud/text-to-speech"); +const fs = require("fs"); +const util = require("util"); +const db = require("../db/database"); +const fse = require("fs-extra"); +const { getAudioDurationInSeconds } = require("get-audio-duration"); +const rimraf = require("rimraf"); + +async function getMp3DurationForText(inputText) { + try { + const client = new textToSpeech.TextToSpeechClient(); + const request = { + input: { text: inputText }, + voice: { languageCode: "en-US", ssmlGender: "NEUTRAL" }, + audioConfig: { audioEncoding: "MP3" } + }; + const [response] = await client.synthesizeSpeech(request); + const writeFile = util.promisify(fs.writeFile); + const fileNamePath = "public/audio/output.mp3"; + await writeFile(fileNamePath, response.audioContent, "binary"); + console.log("Audio content written to file: output.mp3"); + + let duration = await getAudioDurationInSeconds(fileNamePath); + + return duration; + } catch (error) { + console.log(error); + return -1; + } +} + +async function getUniqueMp3DurationForText(inputText) { + try { + const client = new textToSpeech.TextToSpeechClient(); + const request = { + input: { text: inputText }, + voice: { languageCode: "en-US", ssmlGender: "NEUTRAL" }, + audioConfig: { audioEncoding: "MP3" } + }; + const [response] = await client.synthesizeSpeech(request); + const writeFile = util.promisify(fs.writeFile); + const uuid = await create_UUID(); + const fileNamePath = `public/audio/unique/output_${uuid}.mp3`; + await writeFile(fileNamePath, response.audioContent, "binary"); + console.log("Audio content written to file: output.mp3"); + + let duration = await getAudioDurationInSeconds(fileNamePath); + + return duration; + } catch (error) { + console.log(error); + return -1; + } +} + +async function create_UUID() { + var dt = new Date().getTime(); + var uuid = "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, function( + c + ) { + var r = (dt + Math.random() * 16) % 16 | 0; + dt = Math.floor(dt / 16); + return (c == "x" ? r : (r & 0x3) | 0x8).toString(16); + }); + return uuid; +} + +/** Generates mp3 file based on text. Audio is stored in output.mp3 */ +async function generateMp3ForText(inputText) { + try { + const client = new textToSpeech.TextToSpeechClient(); + const request = { + input: { text: inputText }, + voice: { languageCode: "en-US", ssmlGender: "NEUTRAL" }, + audioConfig: { audioEncoding: "MP3" } + }; + const [response] = await client.synthesizeSpeech(request); + const writeFile = util.promisify(fs.writeFile); + await writeFile("public/audio/output.mp3", response.audioContent, "binary"); + console.log("Audio content written to file: output.mp3"); + return true; + } catch (error) { + console.log(error); + return false; + } +} + +async function generateMp3ForTextInFileystem(fileNamePath, inputText) { + try { + const client = new textToSpeech.TextToSpeechClient(); + const request = { + input: { text: inputText }, + voice: { languageCode: "en-US", ssmlGender: "NEUTRAL" }, + audioConfig: { audioEncoding: "MP3" } + }; + const [response] = await client.synthesizeSpeech(request); + const writeFile = util.promisify(fse.outputFile); + await writeFile(fileNamePath, response.audioContent, "binary"); + console.log(`Audio content written to file: ${fileNamePath}`); + return true; + } catch (error) { + console.log(error); + return false; + } +} + +async function generateAudioDescriptionsInDbWithInfo( + vid_id, + vol_id, + scene_id, + res, + merge = false +) { + /** Create sentences from the scene description */ + let res1 = await generateSentencesForDescriptionsInDbWithInfo( + vid_id, + vol_id, + scene_id + ); + + /** Calculate audio durations for the sentences */ + let res2 = await generateMp3ForSentencesInDbWithInfo( + vid_id, + vol_id, + scene_id, + merge + ); + + /** Insert inline/extended audio descriptions in the video */ + let res3 = await generateTimestampsforSentencesInDbWithInfo( + vid_id, + vol_id, + scene_id + ); + if (!merge) { + await getResponse(vid_id, vol_id, scene_id, res); + } else { + /** Merge sentences and audio clips of the sentences based on conitnuity */ + let res4 = await mergeSentenceAudios(vid_id, vol_id); + + return res1 && res2 && res3 && res4; + } +} + +/** Step 1: Creates separate sentences of given descriptions in the db for all videos & users */ +async function generateSentencesForDescriptionsInDbWithInfo( + vid_id, + vol_id, + scene_id +) { + try { + await db.any( + `Delete from descriptions_sentence where video_id = '${vid_id}' and volunteer_id = '${vol_id}' and scene_id = '${scene_id}'` + ); + } catch (error) { + console.log(error); + return false; + } + + try { + let descriptionListArr = await db.any( + `SELECT * FROM descriptions WHERE video_id = '${vid_id}' AND volunteer_id = '${vol_id}' AND scene_id = '${scene_id}' order by id` + ); + for (let i = 0; i < descriptionListArr.length; i++) { + let currDescription = descriptionListArr[i]; + let currDescriptionId = currDescription.desc_id; + let currVideoId = currDescription.video_id; + let currSceneId = currDescription.scene_id; + let currScene_start_time = currDescription.start_time; + let currScene_end_time = currDescription.end_time; + let volunteerId = currDescription.volunteer_id; + let hasAi = currDescription.has_ai; + let descriptionText = currDescription.description; + let ocrText = currDescription.ocr; + let sceneNum = currDescription.scene_num; + ocrText = + hasAi && ocrText && ocrText.trim().length > 0 + ? "\nText on screen : " + ocrText + : ""; + let lastChar = descriptionText[descriptionText.length - 1]; + if ( + descriptionText.length > 0 && + lastChar != "." && + lastChar != "!" && + lastChar != "?" && + lastChar != "\n" + ) + descriptionText += "."; + let finalText = descriptionText + ocrText; + let descType = hasAi ? "ai" : "non_ai"; + sentences = finalText + .replace(/([.?!\n])\s*(?=[A-Za-z])/g, "$1|") + .split("|"); + let sentence_id_key = 0; + let ocr_flag = true; + is_ocr = false; + + for (s of sentences) { + if ( + ocr_flag && + s.length > 18 && + s.substring(0, 17) == "Text on screen : " + ) { + is_ocr = true; + ocr_flag = false; + } + let sentence_id = currDescriptionId + "_" + sentence_id_key; + await insertDescriptionSentencesTable( + currDescriptionId, + currVideoId, + currSceneId, + currScene_start_time, + currScene_end_time, + volunteerId, + hasAi, + descriptionText, + ocrText, + sceneNum, + finalText, + sentence_id, + s, + is_ocr + ); + sentence_id_key += 1; + } + } + return true; + } catch (error) { + console.log(error); + return false; + } +} + +async function insertDescriptionSentencesTable( + currDescriptionId, + currVideoId, + currSceneId, + currScene_start_time, + currScene_end_time, + volunteerId, + hasAi, + descriptionText, + ocrText, + sceneNum, + finalText, + sentence_id, + s, + is_ocr +) { + try { + await db.query(`INSERT INTO descriptions_sentence(desc_id,video_id,scene_id, + start_time,end_time,volunteer_id,has_ai,description,ocr, + scene_num,sentence_id,sentence, is_ocr) VALUES('${currDescriptionId}','${currVideoId}','${currSceneId}', + ${currScene_start_time},${currScene_end_time},'${volunteerId}',${hasAi},'${descriptionText}','${ocrText}', + ${sceneNum},'${sentence_id}','${s}', ${is_ocr});`); + + return true; + } catch (e) { + console.log(e); + return false; + } +} + +async function updateSentencesDescriptionTable( + duration, + currVideoId, + volunteerId, + hasAi, + sceneNum, + relativePath, + sentence_id +) { + try { + await db.query( + `UPDATE descriptions_sentence SET audio_length = ${duration}, audiopath = '${relativePath}' where video_id = '${currVideoId}' and volunteer_id = '${volunteerId}' and has_ai=${hasAi} and scene_num=${sceneNum} and sentence_id='${sentence_id}'` + ); + return true; + } catch (e) { + console.log(e); + return false; + } +} + +/** Step 2: Generates mp3 audio files for all the sentences */ +async function generateMp3ForSentencesInDbWithInfo( + vid_id, + vol_id, + scene_id, + merge = false +) { + try { + let sentenceListArr = await db.any( + `SELECT * FROM descriptions_sentence WHERE video_id = '${vid_id}' AND volunteer_id = '${vol_id}' AND scene_id = '${scene_id}' order by scene_num, sentence_id` + ); + + let scene_number = sentenceListArr[0].scene_num; + + rimraf.sync(`public/audio/${vid_id}/ai/${vol_id}/${scene_number}`); + + for (let i = 0; i < sentenceListArr.length; i++) { + let currSentence = sentenceListArr[i]; + let currDescriptionId = currSentence.desc_id; + let currVideoId = currSentence.video_id; + let currSceneId = currSentence.scene_id; + let currScene_start_time = currSentence.start_time; + let currScene_end_time = currSentence.end_time; + let volunteerId = currSentence.volunteer_id; + let hasAi = currSentence.has_ai; + let descriptionText = currSentence.description; + let ocrText = currSentence.ocr; + let sceneNum = currSentence.scene_num; + ocrText = + hasAi && ocrText && ocrText.trim().length > 0 + ? "\nText on screen : " + ocrText + : ""; + let lastChar = descriptionText[descriptionText.length - 1]; + if ( + lastChar != null && + lastChar != "." && + lastChar != "!" && + lastChar != "?" + ) + descriptionText += "."; + let finalText = descriptionText + ocrText; + let descType = hasAi ? "ai" : "non_ai"; + let sent = currSentence.sentence; + let str = currSentence.sentence_id; + let newstr = str.split("_"); + let sentNum = newstr[newstr.length - 1]; + let fileNamePath = ""; + if (merge) + fileNamePath = `public/audio/${currVideoId}/${descType}/${volunteerId}/${sceneNum}/${sentNum}.mp3`; + else + fileNamePath = `public/audio/${currVideoId}/${descType}/${volunteerId}/${sceneNum}/${sentNum}.mp3`; + + let relativePath = fileNamePath.replace("public", ".."); + + await generateMp3ForTextInFileystem(fileNamePath, sent); + let duration = await getAudioDurationInSeconds(fileNamePath); + console.log(duration); + await updateSentencesDescriptionTable( + duration, + currVideoId, + volunteerId, + hasAi, + sceneNum, + relativePath, + str + ); + } + return true; + } catch (error) { + console.log(error); + return false; + } +} +/** API for generating timestamps for audio descriptions for a scene */ +async function generateTimestampsforScene( + dbVideoId, + dbVolId, + sceneId, + description +) { + let sentences = description + .replace(/([.?!\n])\s*(?=[A-Za-z])/g, "$1|") + .split("|"); + let sentence_duration = []; + + for (let s of sentences) { + let dur = await getUniqueMp3DurationForText(s); + sentence_duration.push(dur); + } + + let sceneInfo = await db.query( + `SELECT * FROM descriptions WHERE video_id='${dbVideoId}' and volunteer_id='${dbVolId}' and scene_id='${sceneId}'` + ); + let size = 1; + // let scene_duration = scene_end_time - scene_start_time; + let scene_start_time = sceneInfo[0].start_time; + let scene_end_time = sceneInfo[0].end_time; + let scene_num = sceneInfo[0].scene_num; + + let dialog_timestamps = await db.query( + `SELECT * FROM video_dialogs WHERE video_id = '${dbVideoId}' ORDER BY start_time` + ); + let currDialogStartTimes = []; + let currDialogEndTimes = []; + let len = dialog_timestamps.length - 1; + + for (let size = 0; size <= len; size++) { + currDialogStartTimes.push(dialog_timestamps[size]["start_time"]); + currDialogEndTimes.push(dialog_timestamps[size]["end_time"]); + } + + while (currDialogEndTimes[len] > scene_end_time) { + len--; + } + + let s = sentences.length - 1; + let flag = true; + let time_gap = 0; + if (len >= 0) time_gap = scene_end_time - currDialogEndTimes[len]; + else time_gap = scene_end_time - scene_start_time; + let time_end = scene_end_time - 0.1; + let time_start = time_end - sentence_duration[s]; + let res = []; + + while ( + len >= 0 && + (flag || currDialogStartTimes[len] > scene_start_time) && + s >= 0 && + time_gap > -1 + ) { + if (time_gap > sentence_duration[s]) { + let obj = {}; + obj["sentence"] = sentences[s]; + obj["audio_duration"] = sentence_duration[s]; + obj["start time"] = time_start.toFixed(3); + obj["end time"] = time_end.toFixed(3); + obj["audio_type"] = "inline"; + res.push(obj); + s--; + if (s < 0) break; + + time_gap -= sentence_duration[s]; + time_end = time_start - 0.1; + time_start = time_end - sentence_duration[s]; + // flag = false; + continue; + } + len--; + if (len < 0) break; + time_gap = currDialogStartTimes[len + 1] - currDialogEndTimes[len]; + time_end = currDialogStartTimes[len + 1] - 0.1; + time_start = time_end - sentence_duration[s]; + flag = false; + } + + while (s >= 0) { + let obj = {}; + obj["sentence"] = sentences[s]; + obj["audio_duration"] = sentence_duration[s]; + obj["start time"] = scene_start_time; + obj["end time"] = "N/A"; + obj["audio_type"] = "extended"; + res.push(obj); + s--; + } + + return res; +} + +/** Step 3: Insert audio descriptions inline/extended in the video for a particular user and video*/ +async function generateTimestampsforSentencesInDbWithInfo( + dbVideoId, + dbVolId, + scene_id +) { + try { + console.log("Video_id is: " + dbVideoId); + let sentences = await db.query( + `SELECT * FROM descriptions_sentence WHERE video_id='${dbVideoId}' and volunteer_id='${dbVolId}' and scene_id='${scene_id}' ORDER BY scene_num, sentence_id` + ); + let size = 0; + let sentence_duration = []; + let scene_start_time = []; + let sentence_video_id = []; + let sentence_video_end_times = []; + let sentence_id = []; + let sentence_scene = []; + + while (size < sentences.length) { + let currSentence = sentences[size]; + sentence_id.push(currSentence.sentence_id); + sentence_duration.push(currSentence.audio_length); + scene_start_time.push(currSentence.start_time); + sentence_video_id.push(currSentence.video_id); + sentence_video_end_times.push(currSentence.end_time); + sentence_scene.push(currSentence.scene_num); + size++; + } + + // Call fn for generating time gaps + await generateTimeGaps(dbVideoId, dbVolId); + + let time_gaps_info = await db.query( + `SELECT * FROM video_time_gaps WHERE video_id = '${dbVideoId}' and volunteer_id = '${dbVolId}'` + ); + + let time_gaps_start_time = []; + let time_gaps_end_time = []; + let time_gaps_scene_num = []; + let time_gaps_is_overlap = []; + + for (let item = 0; item < time_gaps_info.length; item++) { + time_gaps_data = time_gaps_info[item]; + time_gaps_start_time.push(time_gaps_data.start_time); + time_gaps_end_time.push(time_gaps_data.end_time); + time_gaps_scene_num.push(time_gaps_data.scene_num); + time_gaps_is_overlap.push(time_gaps_data.is_overlap); + } + let i = 0, + j = 0; + let currScene = sentence_scene[0]; + + while (i < sentences.length && j < time_gaps_info.length) { + /** Extended */ + if (time_gaps_scene_num[j] > sentence_scene[i]) { + let presentSceneNum = sentence_scene[i]; + + while (presentSceneNum == sentence_scene[i]) { + let sceneEndTime = sentence_video_end_times[i]; + + await updateSentenceStartTime( + sentence_id[i], + sceneEndTime, + "extended" + ); + i++; + } + } + + currScene = sentence_scene[i]; + + while (j < time_gaps_info.length && time_gaps_scene_num[j] < currScene) + j++; + + let time_start = time_gaps_start_time[j]; // May have to add error '0.01' + let time_end = time_gaps_end_time[j]; + + if (j != 0 && time_gaps_is_overlap[j - 1]) + time_start = scene_start_time[i]; + + let time_gap = time_end - time_start; + + // let end_time_sceneNum = await fetchSceneNum(time_end, currVideo, sentence_id[i]); + // let start_time_sceneNum = await fetchSceneNum(time_start, currVideo, sentence_id[i]); + + while ( + i < sentences.length && + time_gap > sentence_duration[i] && + currScene == sentence_scene[i] && + time_gaps_scene_num[j] == sentence_scene[i] + ) { + await updateSentenceStartTime(sentence_id[i], time_start, "inline"); + time_gap -= sentence_duration[i] + 0.1; + i++; + + if (i >= sentences.length) { + break; + } + + time_start += sentence_duration[i - 1] + 0.1; + } + + j++; + } + + while (i < sentences.length) { + let presentSceneNum = sentence_scene[i]; + + let sceneEndTime = sentence_video_end_times[i]; + await updateSentenceStartTime(sentence_id[i], sceneEndTime, "extended"); + i++; + } + } catch (error) { + console.log(error); + return false; + } +} + +async function generateTimeGaps(vid_id, vol_id) { + try { + await db.any( + `Delete from video_time_gaps where video_id = '${vid_id}' and volunteer_id = '${vol_id}'` + ); + } catch (error) { + console.log(error); + return false; + } + + // try{ + + // let res = await db.any(`Select * FROM video_time_gaps where video_id = '${vid_id}'`); + // if(res.length > 0) + // return true; + + // } catch (error) { + // console.log(error); + // return false; + // } + + let descriptionListArr = await db.any( + `SELECT * FROM descriptions WHERE video_id = '${vid_id}' AND volunteer_id = '${vol_id}'order by scene_num` + ); + + let scenes = {}; + + for (let i = 0; i < descriptionListArr.length; i++) { + let currDescription = descriptionListArr[i]; + times = {}; + times["start_time"] = currDescription.start_time; + times["end_time"] = currDescription.end_time; + times["video_dialogs"] = []; + + scenes[currDescription.scene_num] = times; + } + + let video_dialogs = await db.any( + `SELECT * FROM video_dialogs WHERE video_id = '${vid_id}' order by start_time` + ); + + for (let i = 0; i < video_dialogs.length; i++) { + let currDialog = video_dialogs[i]; + + let cd_startTime = currDialog.start_time; + let cd_endTime = currDialog.end_time; + let begin = await fetchSceneNum(cd_startTime, vid_id, vol_id + "_parse"); + let end = await fetchSceneNum(cd_endTime, vid_id, vol_id); + + if (begin == end) { + let vdObj = {}; + vdObj["start_time"] = cd_startTime; + vdObj["end_time"] = cd_endTime; + + scenes[begin]["video_dialogs"].push(vdObj); + } else { + let i = begin; + + while (i <= end) { + let begin_time = 0, + end_time = 0; + let vdObj = {}; + + if (!scenes.hasOwnProperty(i)) { + i++; + continue; + } + + if (i == begin) begin_time = cd_startTime; + else begin_time = scenes[i]["start_time"]; + + if (i == end) end_time = cd_endTime; + else end_time = scenes[i]["end_time"]; + + vdObj["start_time"] = begin_time; + vdObj["end_time"] = end_time; + scenes[i]["video_dialogs"].push(vdObj); + i++; + } + } + } + + for (scene of Object.keys(scenes)) { + let video_dialogs = scenes[scene]["video_dialogs"]; + let scene_num = scene; + let scene_startTime = scenes[scene]["start_time"]; + let scene_endTime = scenes[scene]["end_time"]; + let is_overlap = false; + let i = 0, + len = video_dialogs.length; + + // if(len == 0){ + // await db.query(`INSERT INTO video_time_gaps(video_id, scene_num, + // start_time, end_time, is_diff_scene) VALUES ('${vid_id}', ${scene_num}, ${scene_startTime}, ${scene_endTime}, ${is_overlap})`); + // } + + while (i <= len) { + let time_start = 0, + time_end = 0; + + if (i == 0) time_start = scene_startTime; + else time_start = video_dialogs[i - 1]["end_time"]; + + if (i == len) time_end = scene_endTime; + else time_end = video_dialogs[i]["start_time"]; + + if (time_start !== time_end) + await db.query(`INSERT INTO video_time_gaps(video_id, volunteer_id, scene_num, + start_time, end_time, is_diff_scene) VALUES ('${vid_id}', '${vol_id}', ${scene_num}, ${time_start}, ${time_end}, ${is_overlap})`); + i++; + } + } +} + +/** Fetch scene num from a given timestamp, videoId */ +async function fetchSceneNum(timestamp, videoId, sentence_id) { + // console.log('Scene timestamps: '+timestamp+' and '+videoId+' and '+sentence_id); + let newstr = sentence_id.split("_"); + let volunteerId = newstr[0]; + + let sceneStartTimesArr = await db.query( + `SELECT start_time FROM descriptions_sentence WHERE video_id = '${videoId}' AND volunteer_id = '${volunteerId}' ORDER BY scene_num` + ); + let sceneNumArr = await db.query( + `SELECT scene_num FROM descriptions_sentence WHERE video_id = '${videoId}' AND volunteer_id = '${volunteerId}' ORDER BY scene_num` + ); + // console.log('Scene timestamps: '+sceneStartTimesArr+' and '+sceneNumArr); + if (timestamp < 0) return -1; + + // for(let i=0; i timestamp) + // return sceneNumArr[i-1]['scene_num']; + // } + + // return sceneNumArr[sceneNumArr.length - 1]['scene_num']; + + let start = 0; + let end = sceneNumArr.length - 1; + let index = -1; + while (start <= end) { + let mid = Math.floor((start + end) / 2); + if (sceneStartTimesArr[mid]["start_time"] == timestamp) { + index = mid; + break; + } else if (sceneStartTimesArr[mid]["start_time"] < timestamp) { + index = mid; + start = mid + 1; + } else { + end = mid - 1; + } + } + // return index; + return sceneNumArr[index]["scene_num"]; +} + +async function updateSentenceStartTime(sentence_id, audioStartTime, audioType) { + console.log( + "Updating sentence timestamp of " + + sentence_id + + " : " + + audioStartTime + + " type: " + + audioType + ); + try { + await db.query( + `UPDATE descriptions_sentence SET audio_start_time = ${audioStartTime}, audio_type = '${audioType}' where sentence_id = '${sentence_id}'` + ); + return true; + } catch (e) { + console.log(e); + return false; + } +} +/** Step 4: Merge all continuous audio clips */ +async function mergeSentenceAudios(video_id, vol_id) { + try { + await db.any( + `Delete from descriptions_new_sentence where video_id = '${video_id}' and volunteer_id = '${vol_id}'` + ); + } catch (error) { + console.log(error); + return false; + } + + try { + let sentenceListArr = await db.any( + `SELECT * FROM descriptions_sentence WHERE video_id = '${video_id}' AND volunteer_id = '${vol_id}' ORDER BY scene_num, sentence_id` + ); + let size = 0; + let sentence = []; + let sentence_id = []; + let sentence_desc = []; + let sentence_ocr = []; + let sentence_duration = []; + let scene_start_time = []; + let scene_end_time = []; + let sentence_video_id = []; + let sentence_video_end_times = []; + let sentence_scene = []; + let sentence_audio_type = []; + let sentence_audio_start_time = []; + let sentence_audio_path = []; + + while (size < sentenceListArr.length) { + let currSentence = sentenceListArr[size]; + sentence.push(currSentence.sentence); + sentence_id.push(currSentence.sentence_id); + sentence_desc.push(currSentence.description); + sentence_ocr.push(currSentence.ocr); + sentence_duration.push(currSentence.audio_length); + scene_start_time.push(currSentence.start_time); + scene_end_time.push(currSentence.end_time); + sentence_video_id.push(currSentence.video_id); + sentence_video_end_times.push(currSentence.end_time); + sentence_scene.push(currSentence.scene_num); + sentence_audio_type.push(currSentence.audio_type); + sentence_audio_start_time.push(currSentence.audio_start_time); + sentence_audio_path.push(currSentence.audiopath); + size++; + } + + let i = 0; + let l = sentenceListArr.length; + let path_val = 0; + + while (i < l) { + let new_sentence = sentence[i]; + let currSceneNum = sentence_scene[i]; + let currSceneStartTime = scene_start_time[i]; + let currSceneEndTime = scene_end_time[i]; + let currSceneDesc = sentence_desc[i] + sentence_ocr[i]; + + let currAudioType = sentence_audio_type[i]; + let audioStartTime = sentence_audio_start_time[i]; + let oldAudioStartTime = audioStartTime; + let oldAudioDur = sentence_duration[i]; + let totalAudioDur = oldAudioDur; + let currAudioPath = sentence_audio_path[i]; + let sentence_id_first = sentence_id[i]; + let sentence_id_last = sentence_id[i]; + + i++; + + while ( + currSceneNum == sentence_scene[i] && + currAudioType == sentence_audio_type[i] && + i < l + ) { + if ( + currAudioType == "inline" && + oldAudioStartTime + oldAudioDur + 0.11 < sentence_audio_start_time[i] + ) + break; + + new_sentence += " " + sentence[i]; + oldAudioStartTime = sentence_audio_start_time[i]; + oldAudioDur = sentence_duration[i]; + totalAudioDur += oldAudioDur; + sentence_id_last = sentence_id[i]; + + i++; + } + await updateNewSentence( + sentence_id_first, + sentence_id_last, + currSceneNum, + currSceneStartTime, + currSceneEndTime, + currSceneDesc, + currAudioType, + audioStartTime, + totalAudioDur, + currAudioPath, + new_sentence, + path_val + ); + + if ( + currSceneNum == sentence_scene[i] && + currAudioType == sentence_audio_type[i] && + i < l + ) + path_val += 1; + else path_val = 0; + } + } catch (error) { + console.log(error); + return false; + } +} + +async function getResponse(vid_id, vol_id, scene_id, res) { + let sentenceListArr = await db.any( + `SELECT * FROM descriptions_sentence WHERE video_id = '${vid_id}' AND volunteer_id = '${vol_id}' AND scene_id = '${scene_id}' order by scene_num, sentence_id` + ); + + let result = {}; + let sceneMap = {}; + let scene_num = sentenceListArr[0].scene_num; + + for (let i = 0; i < sentenceListArr.length; i++) { + let currSentence = sentenceListArr[i]; + let currSentenceId = currSentence.sentence_id; + let currDescriptionId = currSentence.desc_id; + let currVideoId = currSentence.video_id; + let currSceneId = currSentence.scene_id; + let currScene_start_time = currSentence.start_time; + let currScene_end_time = currSentence.end_time; + let volunteerId = currSentence.volunteer_id; + let hasAi = currSentence.has_ai; + let descriptionText = currSentence.description; + let ocrText = currSentence.ocr; + let is_ocr = currSentence.is_ocr; + let sceneNum = currSentence.scene_num; + let currAudioPath = currSentence.audiopath; + let currAudioType = currSentence.audio_type; + let currAudioStartTime = currSentence.audio_start_time; + let currAudioLength = currSentence.audio_length; + let currSent = currSentence.sentence; + + if (!result.hasOwnProperty(currSceneId)) { + result[currSceneId] = {}; + result[currSceneId]["description"] = descriptionText; + result[currSceneId]["ocr"] = ocrText; + result[currSceneId]["sentences"] = []; + result[currSceneId]["ocr sentences"] = []; + result[currSceneId]["empty gaps"] = []; + sceneMap[sceneNum] = currSceneId; + } + + let sentenceObj = {}; + sentenceObj["sentence id"] = currSentenceId; + sentenceObj["sentence"] = currSent; + sentenceObj["audio type"] = currAudioType; + sentenceObj["audio start time"] = currAudioStartTime; + sentenceObj["audio duration"] = currAudioLength; + sentenceObj["url"] = + "http://18.221.192.73:5001" + + currAudioPath.substring(2, currAudioPath.length); + + if (!is_ocr) result[currSceneId]["sentences"].push(sentenceObj); + else result[currSceneId]["ocr sentences"].push(sentenceObj); + } + + let time_gaps_info = await db.query( + `SELECT * FROM video_time_gaps WHERE video_id = '${vid_id}' and volunteer_id = '${vol_id}' AND scene_num = '${scene_num}' ORDER BY scene_num, start_time` + ); + + for (let item = 0; item < time_gaps_info.length; item++) { + time_gaps_data = time_gaps_info[item]; + let time_gaps_start_time = time_gaps_data.start_time; + let time_gaps_end_time = time_gaps_data.end_time; + let time_gaps_scene_num = time_gaps_data.scene_num; + let time_gaps_is_overlap = time_gaps_data.is_overlap; + + gapObj = {}; + gapObj["start time"] = time_gaps_start_time; + gapObj["end time"] = time_gaps_end_time; + let scene_id = sceneMap[time_gaps_scene_num]; + + result[scene_id]["empty gaps"].push(gapObj); + } + // result["dialog timestamps"] = []; + // let dialog_ts_info = await db.query(`SELECT * FROM video_dialogs WHERE video_id = '${vid_id}' ORDER BY start_time`); + + // for(let item = 0; item < dialog_ts_info.length; item++) + // { + // dialog_ts_data = dialog_ts_info[item]; + // let dialog_ts_start_time = dialog_ts_data.start_time; + // let dialog_ts_end_time = dialog_ts_data.end_time; + + // dialogObj = {}; + // dialogObj["start time"] = dialog_ts_start_time; + // dialogObj["end time"] = dialog_ts_end_time; + + // result["dialog timestamps"].push(dialogObj); + + // } + + res.send(result); +} + +async function updateNewSentence( + sentence_id_first, + sentence_id_last, + currSceneNum, + currSceneStartTime, + currSceneEndTime, + currSceneDesc, + currAudioType, + audioStartTime, + totalAudioDur, + currAudioPath, + sentence, + path_val +) { + let newstr = sentence_id_last.split("_"); + let sentence_id = sentence_id_first + "_" + newstr[newstr.length - 1]; + let vol_id = newstr[0]; + let video_id = newstr[1]; + let str = currAudioPath.split("/"); + let audio_path = ""; + let file_path = "public/"; + + for (let i = 0; i < str.length - 1; i++) audio_path += str[i] + "/"; + for (let i = 1; i < str.length - 1; i++) file_path += str[i] + "/"; + + // audio_path = audio_path + 'new/' + currAudioType + '.mp3'; + if (path_val == 0) { + file_path = file_path + "new/" + currAudioType + ".mp3"; + audio_path = audio_path + "new/" + currAudioType + ".mp3"; + } else { + file_path = + file_path + "new/" + currAudioType + "_" + String(path_val) + ".mp3"; + audio_path = + audio_path + "new/" + currAudioType + "_" + String(path_val) + ".mp3"; + } + + console.log( + "******************************************************************************************************************" + ); + console.log("Audio path is: " + audio_path); + console.log("File path is: " + file_path); + + // console.log('sentence_id_first', sentence_id_first); + // console.log('sentence_id_last' + sentence_id_last); + console.log("currSceneNum: " + currSceneNum); + // console.log('currSceneStartTime: ' + currSceneStartTime); + // console.log('currSceneEndTime: ',currSceneEndTime); + // console.log('currSceneDesc' + currSceneDesc); + + // console.log('currAudioType: ' + currAudioType); + // console.log('audioStartTime: ' + audioStartTime); + await generateMp3ForTextInFileystem(file_path, sentence); + let duration = await getAudioDurationInSeconds(file_path); + console.log("duration " + duration); + console.log("totalAudioDur: " + totalAudioDur); + console.log("sentence" + sentence); + console.log( + "******************************************************************************************************************" + ); + + try { + await db.query(`INSERT INTO descriptions_new_sentence(video_id, volunteer_id, scene_num, + start_time,end_time,description,sentence,audio_type,audio_start_time,audio_length,audio_path, + sentence_id) VALUES('${video_id}','${vol_id}',${currSceneNum}, + ${currSceneStartTime},${currSceneEndTime},'${currSceneDesc}','${sentence}','${currAudioType}',${audioStartTime},${duration},'${audio_path}', + '${sentence_id}');`); + + return true; + } catch (e) { + console.log(e); + return false; + } +} + +module.exports = { + generateMp3ForText: generateMp3ForText, + generateAudioDescriptionsInDbWithInfo: generateAudioDescriptionsInDbWithInfo, + generateTimestampsforScene: generateTimestampsforScene, + generateTimestampsforSentencesInDbWithInfo: generateTimestampsforSentencesInDbWithInfo, + mergeSentenceAudios: mergeSentenceAudios +}; diff --git a/backend/audioDescriptions.js b/backend/audioDescriptions.js new file mode 100644 index 0000000..7065731 --- /dev/null +++ b/backend/audioDescriptions.js @@ -0,0 +1,1030 @@ +const textToSpeech = require('@google-cloud/text-to-speech'); +const fs = require('fs'); +const util = require('util'); +const db = require('../db/database'); +const fse = require('fs-extra'); +const { getAudioDurationInSeconds } = require('get-audio-duration') +const rimraf = require("rimraf"); + +async function getMp3DurationForText(inputText) { + try { + const client = new textToSpeech.TextToSpeechClient(); + const request = { + input: {text: inputText}, + voice: {languageCode: 'en-US', ssmlGender: 'NEUTRAL'}, + audioConfig: {audioEncoding: 'MP3', "speakingRate": 1.5}, + }; + const [response] = await client.synthesizeSpeech(request); + const writeFile = util.promisify(fs.writeFile); + const fileNamePath = 'public/audio/output.mp3'; + await writeFile(fileNamePath, response.audioContent, 'binary'); + console.log('Audio content written to file: output.mp3'); + + let duration = await getAudioDurationInSeconds(fileNamePath); + + return duration; + } catch (error) { + console.log(error); + return -1; + } +} + +async function getUniqueMp3DurationForText(inputText) { + try { + const client = new textToSpeech.TextToSpeechClient(); + const request = { + input: {text: inputText}, + voice: {languageCode: 'en-US', ssmlGender: 'NEUTRAL'}, + audioConfig: {audioEncoding: 'MP3'}, + }; + const [response] = await client.synthesizeSpeech(request); + const writeFile = util.promisify(fs.writeFile); + const uuid = await create_UUID(); + const fileNamePath = `public/audio/unique/output_${uuid}.mp3`; + await writeFile(fileNamePath, response.audioContent, 'binary'); + console.log('Audio content written to file: output.mp3'); + + let duration = await getAudioDurationInSeconds(fileNamePath); + + return duration; + } catch (error) { + console.log(error); + return -1; + } +} + +async function create_UUID(){ + var dt = new Date().getTime(); + var uuid = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { + var r = (dt + Math.random()*16)%16 | 0; + dt = Math.floor(dt/16); + return (c=='x' ? r :(r&0x3|0x8)).toString(16); + }); + return uuid; +} + + /** Generates mp3 file based on text. Audio is stored in output.mp3 */ +async function generateMp3ForText(inputText) { + try { + const client = new textToSpeech.TextToSpeechClient(); + const request = { + input: {text: inputText}, + voice: {languageCode: 'en-US', ssmlGender: 'NEUTRAL'}, + audioConfig: {audioEncoding: 'MP3'}, + }; + const [response] = await client.synthesizeSpeech(request); + const writeFile = util.promisify(fs.writeFile); + await writeFile('public/audio/output.mp3', response.audioContent, 'binary'); + console.log('Audio content written to file: output.mp3'); + return true; + } catch (error) { + console.log(error); + return false; + } +} + +async function generateMp3ForTextInFileystem(fileNamePath, inputText, voiceName='en-US-Wavenet-D') { + try { + const client = new textToSpeech.TextToSpeechClient(); + const request = { + input: {text: inputText}, + voice: {languageCode: 'en-US', name: voiceName, ssmlGender: 'NEUTRAL'}, + audioConfig: {audioEncoding: 'MP3', "speakingRate": 1.5}, + }; + const [response] = await client.synthesizeSpeech(request); + const writeFile = util.promisify(fse.outputFile); + await writeFile(fileNamePath, response.audioContent, 'binary'); + console.log(`Audio content written to file: ${fileNamePath}`); + return true; + } catch (error) { + console.log(error); + return false; + } +} + +async function generateMp3ForTextInFileystemWithDifferentVoices(fileNamePath, sentence, sentenceOcr){ + try { + const client = new textToSpeech.TextToSpeechClient(); + const request1 = { + input: {text: sentence}, + voice: {languageCode: 'en-US', name: 'en-US-Wavenet-D', ssmlGender: 'NEUTRAL'}, + audioConfig: {audioEncoding: 'MP3', "speakingRate": 1.5}, + }; + + const request2 = { + input: {text: sentenceOcr}, + voice: {languageCode: 'en-US', name: 'en-US-Wavenet-C', ssmlGender: 'NEUTRAL'}, + audioConfig: {audioEncoding: 'MP3', "speakingRate": 1.5}, + }; + + const [response1] = await client.synthesizeSpeech(request1); + const [response2] = await client.synthesizeSpeech(request2); + + const writeFile = util.promisify(fse.outputFile); + await writeFile(fileNamePath, Buffer.concat([response1.audioContent, response2.audioContent]), 'binary'); + console.log(`Audio content written to file: ${fileNamePath}`); + return true; + } catch (error) { + console.log(error); + return false; + } +} + +async function generateAudioDescriptionsInDbWithInfo(vid_id, vol_id, merge=false, hasOcr=false, res) { + + /** Create sentences from the scene description */ + let res1 = await generateSentencesForDescriptionsInDbWithInfo(vid_id, vol_id); + + /** Calculate audio durations for the sentences */ + let res2 = await generateMp3ForSentencesInDbWithInfo(vid_id, vol_id, merge); + + /** Insert inline/extended audio descriptions in the video */ + let res3 = await generateTimestampsforSentencesInDbWithInfo(vid_id, vol_id, hasOcr); + if(!merge){ + await getResponse(vid_id, vol_id, res); + } + else{ + /** Merge sentences and audio clips of the sentences based on conitnuity */ + let res4 = await mergeSentenceAudios(vid_id, vol_id); + + return res1 && res2 && res3 && res4; + } +} + + + /** Step 1: Creates separate sentences of given descriptions in the db for all videos & users */ +async function generateSentencesForDescriptionsInDbWithInfo(vid_id, vol_id) { + + try{ + + await db.any(`Delete from descriptions_sentence where video_id = '${vid_id}' and volunteer_id = '${vol_id}'`); + + } catch (error) { + console.log(error); + return false; + } + + try { + let descriptionListArr = await db.any(`SELECT * FROM descriptions WHERE video_id = '${vid_id}' AND volunteer_id = '${vol_id}'order by id`); + for(let i=0; i0) ? "\nText on screen : " + ocrText : ""; + let lastChar = descriptionText[descriptionText.length - 1]; + if(descriptionText.length > 0 && lastChar != '.' && lastChar != '!' && lastChar != '?' && lastChar != '\n') + descriptionText += '.'; + let finalText = descriptionText + ocrText; + let descType = hasAi ? 'ai' : 'non_ai'; + if(finalText.length == 0) + continue; + sentences = finalText.replace(/([.?!\n])\s*(?=[A-Za-z])/g, "$1|").split("|") + let sentence_id_key = 0; + let ocr_flag = true; + is_ocr = false; + + for(s of sentences) + { + if(ocr_flag && s.length > 18 && s.substring(0,17) == "Text on screen : "){ + is_ocr = true; + ocr_flag = false; + console.log('s -> ',s); + s = s.substring(17,s.length); + console.log('new s -> ',s); + } + let sentence_id = currDescriptionId + '_' + sentence_id_key; + await insertDescriptionSentencesTable(currDescriptionId,currVideoId,currSceneId, + currScene_start_time,currScene_end_time,volunteerId,hasAi,descriptionText,ocrText, + sceneNum,finalText,sentence_id,s, is_ocr); + sentence_id_key += 1; + } + } + return true; + } catch (error) { + console.log(error); + return false; + } +} + +async function insertDescriptionSentencesTable(currDescriptionId,currVideoId,currSceneId, + currScene_start_time,currScene_end_time,volunteerId,hasAi,descriptionText,ocrText, + sceneNum,finalText,sentence_id,s, is_ocr){ + try { + await db.query(`INSERT INTO descriptions_sentence(desc_id,video_id,scene_id, + start_time,end_time,volunteer_id,has_ai,description,ocr, + scene_num,sentence_id,sentence, is_ocr) VALUES('${currDescriptionId}','${currVideoId}','${currSceneId}', + ${currScene_start_time},${currScene_end_time},'${volunteerId}',${hasAi},'${descriptionText}','${ocrText}', + ${sceneNum},'${sentence_id}','${s}', ${is_ocr});`); + + return true; + } catch(e) { + console.log(e); + return false; + } + } + + +async function updateSentencesDescriptionTable(duration, currVideoId, volunteerId, hasAi, sceneNum, relativePath, sentence_id) { + try { + await db.query(`UPDATE descriptions_sentence SET audio_length = ${duration}, audiopath = '${relativePath}' where video_id = '${currVideoId}' and volunteer_id = '${volunteerId}' and has_ai=${hasAi} and scene_num=${sceneNum} and sentence_id='${sentence_id}'`); + return true; + } catch(e) { + console.log(e); + return false; + } +} + +/** Step 2: Generates mp3 audio files for all the sentences */ +async function generateMp3ForSentencesInDbWithInfo(vid_id, vol_id, merge = false) { + + try{ + let sentenceListArr = await db.any(`SELECT * FROM descriptions_sentence WHERE video_id = '${vid_id}' AND volunteer_id = '${vol_id}' order by scene_num, sentence_id`); + + rimraf.sync(`public/audio/${vid_id}/ai/${vol_id}`); + + for(let i=0; i0) ? "\nText on screen : " + ocrText : ""; + let lastChar = descriptionText[descriptionText.length - 1]; + if(lastChar != null && lastChar != '.' && lastChar != '!' && lastChar != '?') + descriptionText += '.'; + let finalText = descriptionText + ocrText; + let descType = hasAi ? 'ai' : 'non_ai'; + let sent = currSentence.sentence; + let str = currSentence.sentence_id; + let newstr = str.split('_'); + let sentNum = newstr[newstr.length - 1]; + let fileNamePath = ''; + if(merge) + fileNamePath = `public/audio/${currVideoId}/${descType}/${volunteerId}/${sceneNum}/${sentNum}.mp3`; + else + fileNamePath = `public/audio/${currVideoId}/${descType}/${volunteerId}/${sceneNum}/${sentNum}.mp3`; + + let relativePath = fileNamePath.replace('public', '..'); + if(isOcr) + await generateMp3ForTextInFileystem(fileNamePath, sent, 'en-US-Wavenet-C'); + else + await generateMp3ForTextInFileystem(fileNamePath, sent); + let duration = await getAudioDurationInSeconds(fileNamePath); + console.log(duration); + await updateSentencesDescriptionTable(duration, currVideoId, volunteerId, hasAi, sceneNum, relativePath, str); + } + return true; + + } catch(error){ + console.log(error); + return false; + } +} + /** API for generating timestamps for audio descriptions for a scene */ +async function generateTimestampsforScene(dbVideoId, dbVolId, sceneId, description){ + + let sentences = description.replace(/([.?!\n])\s*(?=[A-Za-z])/g, "$1|").split("|"); + let sentence_duration = [] + + for(let s of sentences){ + let dur = await getUniqueMp3DurationForText(s); + sentence_duration.push(dur); + } + + let sceneInfo = await db.query(`SELECT * FROM descriptions WHERE video_id='${dbVideoId}' and volunteer_id='${dbVolId}' and scene_id='${sceneId}'`); + let size = 1; + // let scene_duration = scene_end_time - scene_start_time; + let scene_start_time = sceneInfo[0].start_time; + let scene_end_time = sceneInfo[0].end_time; + let scene_num = sceneInfo[0].scene_num; + + let dialog_timestamps = await db.query(`SELECT * FROM video_dialogs WHERE video_id = '${dbVideoId}' ORDER BY start_time`); + if(dialog_timestamps.length == 0){ + dialog_timestamps = await db.any(`SELECT * FROM video_dialogs WHERE video_id = '${dbVideoId}' order by start_time`); + } + + let currDialogStartTimes = []; + let currDialogEndTimes = []; + let len = dialog_timestamps.length - 1; + + for(let size=0; size <= len; size++) + { + currDialogStartTimes.push(dialog_timestamps[size]['start_time']); + currDialogEndTimes.push(dialog_timestamps[size]['end_time']); + } + + while(currDialogEndTimes[len] > scene_end_time){ + len--; + } + + let s = sentences.length - 1; + let flag = true; + let time_gap = 0; + if(len>=0) + time_gap = scene_end_time - currDialogEndTimes[len]; + else + time_gap = scene_end_time - scene_start_time; + let time_end = scene_end_time - 0.1; + let time_start = time_end - sentence_duration[s]; + let res = []; + + while(len >= 0 && (flag || currDialogStartTimes[len] > scene_start_time) && s >= 0 && time_gap > -1){ + + if( time_gap > sentence_duration[s]){ + let obj = {}; + obj['sentence'] = sentences[s]; + obj['audio_duration'] = sentence_duration[s]; + obj['start time'] = time_start.toFixed(3); + obj['end time'] = time_end.toFixed(3); + obj['audio_type'] = 'inline'; + res.push(obj); + s--; + if(s<0) + break; + + time_gap -= sentence_duration[s]; + time_end = time_start - 0.1; + time_start = time_end - sentence_duration[s]; + // flag = false; + continue; + } + len--; + if(len < 0) + break; + time_gap = currDialogStartTimes[len+1] - currDialogEndTimes[len]; + time_end = currDialogStartTimes[len+1] - 0.1; + time_start = time_end - sentence_duration[s]; + flag = false; + } + + while(s >= 0){ + let obj = {}; + obj['sentence'] = sentences[s]; + obj['audio_duration'] = sentence_duration[s]; + obj['start time'] = scene_start_time; + obj['end time'] = 'N/A'; + obj['audio_type'] = 'extended'; + res.push(obj); + s--; + } + + return res; +} + + /** Step 3: Insert audio descriptions inline/extended in the video for a particular user and video*/ +async function generateTimestampsforSentencesInDbWithInfo(dbVideoId, dbVolId, hasOcr){ + try{ + console.log("Video_id is: "+dbVideoId); + let sentences = await db.query(`SELECT * FROM descriptions_sentence WHERE video_id='${dbVideoId}' and volunteer_id='${dbVolId}' ORDER BY scene_num, sentence_id`); + let size = 0; + let sentence_duration = []; + let scene_start_time = []; + let sentence_video_id = []; + let sentence_video_end_times = []; + let sentence_id = []; + let sentence_scene = []; + + while(size sentence_scene[i]){ + let presentSceneNum = sentence_scene[i]; + + while(presentSceneNum == sentence_scene[i] ){ + let sceneEndTime = sentence_video_end_times[i]; + + await updateSentenceStartTime(sentence_id[i], sceneEndTime, 'extended'); + i++; + } + } + + currScene = sentence_scene[i]; + + while( j < time_gaps_info.length && time_gaps_scene_num[j] < currScene) + j++; + + let time_start = time_gaps_start_time[j]; // May have to add error '0.01' + let time_end = time_gaps_end_time[j]; + + if( j != 0 && time_gaps_is_overlap[j-1]) + time_start = scene_start_time[i]; + + let time_gap = time_end - time_start; + + // let end_time_sceneNum = await fetchSceneNum(time_end, currVideo, sentence_id[i]); + // let start_time_sceneNum = await fetchSceneNum(time_start, currVideo, sentence_id[i]); + + while(i < sentences.length && time_gap > sentence_duration[i] && currScene == sentence_scene[i] && time_gaps_scene_num[j] == sentence_scene[i]){ + + while(k < ocrExtendedTimestampsLen && time_start > ocrExtendedTimestamps[k]['audio_start_time']){ + k++; + } + let ocrCollisionEndTime = time_start + sentence_duration[i]; + let ocrCollisionCheckRes = await checkOcrAndInlineCollision(time_start, ocrCollisionEndTime, ocrExtendedTimestamps, k); + + // if( time_start < ocrExtendedTimestamps[k]['audio_start_time'] && ocrExtendedTimestamps[k]['audio_start_time'] < time_start + sentence_duration[i]) + if (ocrCollisionCheckRes['isCollision']) + { + // time_start = ocrExtendedTimestamp[k]['audio_start_time'] + 0.1; + time_gap -= (ocrCollisionCheckRes['newTimestamp'] - time_start); + time_start = ocrCollisionCheckRes['newTimestamp']; + continue; + } + + await updateSentenceStartTime(sentence_id[i], time_start, 'inline'); + time_gap -= (sentence_duration[i] + 0.1); + i++; + + if(i >= sentences.length){break;} + + time_start += sentence_duration[i-1] + 0.1; + } + + j++; + +} + + while(i < sentences.length){ + let presentSceneNum = sentence_scene[i]; + + let sceneEndTime = sentence_video_end_times[i]; + await updateSentenceStartTime(sentence_id[i], sceneEndTime, 'extended'); + i++; +} + +} catch (error) { + console.log(error); + return false; + } + +} + +async function checkOcrAndInlineCollision(startTime, endTime, ocrExtendedTimestampsArr, k){ + + let isCollision = false; + let newTimestamp = startTime; + let len = ocrExtendedTimestampsArr.length; + + for(let i=k; i 0) +// return true; + +// } catch (error) { +// console.log(error); +// return false; +// } + +let descriptionListArr = await db.any(`SELECT * FROM descriptions WHERE video_id = '${vid_id}' AND volunteer_id = '${vol_id}'order by scene_num`); + +let scenes = {}; +let newStartTimes = []; + +for(let i=0; i timestamp) + // return sceneNumArr[i-1]['scene_num']; + // } + + // return sceneNumArr[sceneNumArr.length - 1]['scene_num']; + + let start = 0; + let end = sceneNumArr.length - 1; + let index = -1; + while(start <= end) { + let mid = Math.floor((start + end)/2); + if(sceneStartTimesArr[mid]['start_time'] == timestamp) { + index = mid; + break; + } else if(sceneStartTimesArr[mid]['start_time'] < timestamp) { + index = mid; + start = mid + 1; + } else { + end = mid - 1; + } + } + // return index; + return sceneNumArr[index]['scene_num']; +} + +async function updateSentenceStartTime(sentence_id, audioStartTime, audioType){ + console.log('Updating sentence timestamp of ' + sentence_id+' : '+audioStartTime+' type: '+audioType); + try { + await db.query(`UPDATE descriptions_sentence SET audio_start_time = ${audioStartTime}, audio_type = '${audioType}' where sentence_id = '${sentence_id}'`); + return true; + } catch(e) { + console.log(e); + return false; + } + +} +/** Step 4: Merge all continuous audio clips */ +async function mergeSentenceAudios(video_id, vol_id){ + + try{ + + await db.any(`Delete from descriptions_new_sentence where video_id = '${video_id}' and volunteer_id = '${vol_id}'`); + + } catch (error) { + console.log(error); + return false; + } + + try{ + let sentenceListArr = await db.any(`SELECT * FROM descriptions_sentence WHERE video_id = '${video_id}' AND volunteer_id = '${vol_id}' ORDER BY scene_num, id`); + let size = 0; + let sentence = []; + let sentence_id = []; + let sentence_desc = []; + let sentence_ocr = []; + let sentence_is_ocr = []; + let sentence_duration = []; + let scene_start_time = []; + let scene_end_time = []; + let sentence_video_id = []; + let sentence_video_end_times = []; + let sentence_scene = []; + let sentence_audio_type = []; + let sentence_audio_start_time = []; + let sentence_audio_path = []; + + while(size0 && new_sentence[new_sentence.length-1] == `\n`) + new_sentence=new_sentence.replace('\n','.'); + + if(new_sentence_ocr.length>0 && new_sentence_ocr[new_sentence_ocr.length-1] == `\n`) + new_sentence_ocr=new_sentence_ocr.replace('\n','.'); + + i++; + + while(currSceneNum == sentence_scene[i] && currAudioType == sentence_audio_type[i] && i < l){ + + if(currAudioType == 'inline' && (oldAudioStartTime + oldAudioDur + 0.11) < sentence_audio_start_time[i] ) + break; + + if(sentence[i].length>0 && sentence[i][sentence[i].length-1] == `\n`) + sentence[i]=sentence[i].replace('\n','.'); + + if(isOcr) + new_sentence_ocr += (' ' + sentence[i]); + else if(!isOcr && !sentence_is_ocr[i]) + new_sentence += (' ' + sentence[i]); + else{ + new_sentence_ocr = sentence[i]; + isOcr = true; + } + + + oldAudioStartTime = sentence_audio_start_time[i]; + oldAudioDur = sentence_duration[i]; + totalAudioDur += oldAudioDur; + sentence_id_last = sentence_id[i]; + + i++; + } + await updateNewSentence(sentence_id_first, sentence_id_last, + currSceneNum, currSceneStartTime, currSceneEndTime, currSceneDesc, + currAudioType, audioStartTime, totalAudioDur, currAudioPath, new_sentence, new_sentence_ocr, path_val); + + if(currSceneNum == sentence_scene[i] && currAudioType == sentence_audio_type[i] && i < l) + path_val += 1; + else + path_val = 0; + + } + +} catch (error) { + console.log(error); + return false; +} +} + +async function getResponse(vid_id, vol_id, res){ + + let sentenceListArr = await db.any(`SELECT * FROM descriptions_sentence WHERE video_id = '${vid_id}' AND volunteer_id = '${vol_id}' order by scene_num, sentence_id`); + + let result = {}; + + let sceneMap = {}; + + for(let i=0; i 0) + // return true; + + // } catch (error) { + // console.log(error); + // return false; + // } + + let descriptionListArr = await db.any( + `SELECT * FROM descriptions WHERE video_id = '${vid_id}' AND volunteer_id = '${vol_id}'order by scene_num` + ); + + let scenes = {}; + + for (let i = 0; i < descriptionListArr.length; i++) { + let currDescription = descriptionListArr[i]; + times = {}; + times["start_time"] = currDescription.start_time; + times["end_time"] = currDescription.end_time; + times["video_dialogs"] = []; + + scenes[currDescription.scene_num] = times; + } + let video_dialogs = []; + if (isInitial) + video_dialogs = await db.any( + `SELECT * FROM video_dialogs WHERE video_id = '${vid_id}' order by start_time` + ); + else { + for (let i = 0; i < dialogStartTimeArr.length; i++) { + let newObj = {}; + newObj["start_time"] = dialogStartTimeArr[i]; + newObj["end_time"] = dialogEndTimeArr[i]; + video_dialogs.push(newObj); + } + } + + for (let i = 0; i < video_dialogs.length; i++) { + let currDialog = video_dialogs[i]; + + let cd_startTime = currDialog.start_time; + let cd_endTime = currDialog.end_time; + let begin = await fetchSceneNum(cd_startTime, vid_id, vol_id); + let end = await fetchSceneNum(cd_endTime, vid_id, vol_id); + + if (begin == end) { + let vdObj = {}; + vdObj["start_time"] = cd_startTime; + vdObj["end_time"] = cd_endTime; + + scenes[begin]["video_dialogs"].push(vdObj); + } else { + let i = begin; + + while (i <= end) { + let begin_time = 0, + end_time = 0; + let vdObj = {}; + + if (!scenes.hasOwnProperty(i)) { + i++; + continue; + } + + if (i == begin) begin_time = cd_startTime; + else begin_time = scenes[i]["start_time"]; + + if (i == end) end_time = cd_endTime; + else end_time = scenes[i]["end_time"]; + + vdObj["start_time"] = begin_time; + vdObj["end_time"] = end_time; + scenes[i]["video_dialogs"].push(vdObj); + i++; + } + } + } + + for (scene of Object.keys(scenes)) { + let video_dialogs = scenes[scene]["video_dialogs"]; + let scene_num = scene; + let scene_startTime = scenes[scene]["start_time"]; + let scene_endTime = scenes[scene]["end_time"]; + let is_overlap = false; + let i = 0, + len = video_dialogs.length; + + // if(len == 0){ + // await db.query(`INSERT INTO video_time_gaps(video_id, scene_num, + // start_time, end_time, is_diff_scene) VALUES ('${vid_id}', ${scene_num}, ${scene_startTime}, ${scene_endTime}, ${is_overlap})`); + // } + + while (i <= len) { + let time_start = 0, + time_end = 0; + + if (i == 0) time_start = scene_startTime; + else time_start = video_dialogs[i - 1]["end_time"]; + + if (i == len) time_end = scene_endTime; + else time_end = video_dialogs[i]["start_time"]; + + if (time_start !== time_end) + await db.query(`INSERT INTO video_time_gaps(video_id, volunteer_id, scene_num, + start_time, end_time, is_diff_scene) VALUES ('${vid_id}', '${vol_id}', ${scene_num}, ${time_start}, ${time_end}, ${is_overlap})`); + i++; + } + } +} + +/** Fetch scene details (scene num, start time, end time) from a given timestamp, videoId */ +async function fetchSceneDetails(timestamp, videoId, sentence_id) { + console.log( + "Scene timestamps: " + + timestamp + + " and " + + videoId + + " and " + + sentence_id + ); + let newstr = sentence_id.split("_"); + let volunteerId = newstr[0]; + + let sceneStartTimesArr = await db.query( + `SELECT start_time FROM descriptions WHERE video_id = '${videoId}' AND volunteer_id = '${volunteerId}' ORDER BY scene_num` + ); + let sceneNumArr = await db.query( + `SELECT scene_num, start_time, end_time FROM descriptions WHERE video_id = '${videoId}' AND volunteer_id = '${volunteerId}' ORDER BY scene_num` + ); + // console.log('Scene timestamps: '+sceneStartTimesArr+' and '+sceneNumArr); + if (timestamp < 0) return -1; + + // for(let i=0; i timestamp) + // return sceneNumArr[i-1]['scene_num']; + // } + + // return sceneNumArr[sceneNumArr.length - 1]['scene_num']; + + let start = 0; + let end = sceneNumArr.length - 1; + let index = -1; + while (start <= end) { + let mid = Math.floor((start + end) / 2); + if (sceneStartTimesArr[mid]["start_time"] == timestamp) { + index = mid; + break; + } else if (sceneStartTimesArr[mid]["start_time"] < timestamp) { + index = mid; + start = mid + 1; + } else { + end = mid - 1; + } + } + let sceneInfo = { + sceneNum: sceneNumArr[index]["scene_num"], + startTime: sceneNumArr[index]["start_time"], + endTime: sceneNumArr[index]["end_time"] + }; + + return sceneInfo; +} + +/** Fetch scene num from a given timestamp, videoId */ +async function fetchSceneNum(timestamp, videoId, sentence_id) { + console.log( + "Scene timestamps: " + + timestamp + + " and " + + videoId + + " and " + + sentence_id + ); + let newstr = sentence_id.split("_"); + let volunteerId = newstr[0]; + + let sceneStartTimesArr = await db.query( + `SELECT start_time FROM descriptions WHERE video_id = '${videoId}' AND volunteer_id = '${volunteerId}' ORDER BY scene_num` + ); + let sceneNumArr = await db.query( + `SELECT scene_num FROM descriptions WHERE video_id = '${videoId}' AND volunteer_id = '${volunteerId}' ORDER BY scene_num` + ); + // console.log('Scene timestamps: '+sceneStartTimesArr+' and '+sceneNumArr); + if (timestamp < 0) return -1; + + // for(let i=0; i timestamp) + // return sceneNumArr[i-1]['scene_num']; + // } + + // return sceneNumArr[sceneNumArr.length - 1]['scene_num']; + + let start = 0; + let end = sceneNumArr.length - 1; + let index = -1; + while (start <= end) { + let mid = Math.floor((start + end) / 2); + if (sceneStartTimesArr[mid]["start_time"] == timestamp) { + index = mid; + break; + } else if (sceneStartTimesArr[mid]["start_time"] < timestamp) { + index = mid; + start = mid + 1; + } else { + end = mid - 1; + } + } + // return index; + return sceneNumArr[index]["scene_num"]; +} + +async function fetchSceneEndtime(timestamp, videoId, sentence_id) { + console.log( + "Scene timestamps: " + + timestamp + + " and " + + videoId + + " and " + + sentence_id + ); + let newstr = sentence_id.split("_"); + let volunteerId = newstr[0]; + + let sceneStartTimesArr = await db.query( + `SELECT start_time FROM descriptions WHERE video_id = '${videoId}' AND volunteer_id = '${volunteerId}' ORDER BY scene_num` + ); + let sceneNumArr = await db.query( + `SELECT end_time FROM descriptions WHERE video_id = '${videoId}' AND volunteer_id = '${volunteerId}' ORDER BY scene_num` + ); + // console.log('Scene timestamps: '+sceneStartTimesArr+' and '+sceneNumArr); + if (timestamp < 0) return -1; + + // for(let i=0; i timestamp) + // return sceneNumArr[i-1]['scene_num']; + // } + + // return sceneNumArr[sceneNumArr.length - 1]['scene_num']; + + let start = 0; + let end = sceneNumArr.length - 1; + let index = -1; + while (start <= end) { + let mid = Math.floor((start + end) / 2); + if (sceneStartTimesArr[mid]["start_time"] == timestamp) { + index = mid; + break; + } else if (sceneStartTimesArr[mid]["start_time"] < timestamp) { + index = mid; + start = mid + 1; + } else { + end = mid - 1; + } + } + // return index; + return sceneNumArr[index]["end_time"]; +} + +async function generateTimestampsForOCR(vid_id, vol_id) { + // Call fn for generating time gaps + //await generateTimeGaps(vid_id, vol_id); + + let time_gaps_info = await db.any( + `SELECT * FROM video_dialogs WHERE video_id = '${vid_id}' order by start_time` + ); + + // let time_gaps_info = await db.query(`SELECT * FROM video_time_gaps WHERE video_id = '${vid_id}' and volunteer_id = '${vol_id}'`); + + let time_gaps_start_time = []; + let time_gaps_end_time = []; + let time_gaps_scene_num = []; + let time_gaps_is_overlap = []; + + for (let item = 0; item < time_gaps_info.length; item++) { + time_gaps_data = time_gaps_info[item]; + time_gaps_start_time.push(time_gaps_data.start_time); + time_gaps_end_time.push(time_gaps_data.end_time); + // time_gaps_scene_num.push(time_gaps_data.scene_num); + // time_gaps_is_overlap.push(time_gaps_data.is_overlap); + } + + let video_start_time = await fetchVideoStartTime(vid_id, vol_id); + let video_end_time = await fetchVideoEndTime(vid_id, vol_id); + // let video_start_time = Math.max(...video_end_times); + + let ocr_info = await db.query( + `SELECT * FROM descriptions_ocr WHERE video_id = '${vid_id}' and volunteer_id = '${vol_id}' ORDER BY timestamp` + ); + + let ocr_id = []; + let ocr_text = []; + let ocr_timestamp = []; + let ocr_time_duration = []; + + for (let item = 0; item < ocr_info.length; item++) { + ocr_data = ocr_info[item]; + ocr_id.push(ocr_data.ocr_id); + ocr_text.push(ocr_data.text); + ocr_timestamp.push(ocr_data.timestamp); + ocr_time_duration.push(ocr_data.audio_duration); + } + + let iter_idx = 0; + let flag = true; + let time_start = 0; + let time_end = 0; + let time_gap = 0; + let inline_times = []; + + for (let i = 0; i < ocr_info.length; i++) { + time_gaps_start_time.sort(function(a, b) { + return a - b; + }); + time_gaps_end_time.sort(function(a, b) { + return a - b; + }); + + // let sceneEndTime1 = await fetchSceneEndtime(ocr_timestamp[i], vid_id, vol_id); + let sceneInfo = await fetchSceneDetails(ocr_timestamp[i], vid_id, vol_id); + let sceneEndTime1 = sceneInfo.startTime; + let sceneEndTime2 = sceneInfo.endTime; + + resObj = await fetchNearestDialogInfo( + ocr_timestamp[i], + ocr_time_duration[i], + time_gaps_start_time, + time_gaps_end_time, + video_end_time, + sceneEndTime1, + sceneEndTime2 + ); + + if (resObj["isInline"]) { + let startTime = resObj["start_time"]; + let endTime = resObj["start_time"] + ocr_time_duration[i]; + + time_gaps_start_time.push(startTime); + time_gaps_end_time.push(endTime + 0.5); + + // Insert in db as inline at ocr_timestamp[i] as start time + await updateOCRTableAudioType(vid_id, ocr_id[i], startTime, "inline"); + // Insert in db (new table) as video_dialog + let obj = { start_time: startTime, end_time: endTime }; + inline_times.push(obj); + } else { + // Insert in db as extended at time_end + await updateOCRTableAudioType( + vid_id, + ocr_id[i], + resObj["start_time"], + "extended" + ); + // Insert in db (new table) as scene_end_time (corner case: if we've to play OCR and description together as extended) + } + flag = false; + } + + insertIntoVideoDialogsAndOCR(vid_id, vol_id, inline_times); +} + +async function updateOCRTableAudioType(vid_id, ocrId, startTime, audioType) { + try { + await db.query( + `UPDATE descriptions_ocr SET audio_start_time = ${startTime}, audio_type = '${audioType}' where video_id = '${vid_id}' and ocr_id='${ocrId}'` + ); + return true; + } catch (e) { + console.log(e); + return false; + } +} + +async function insertIntoVideoDialogsAndOCR(vid_id, vol_id, inline_times) { + let video_dialogs = await db.any( + `SELECT * FROM video_dialogs WHERE video_id = '${vid_id}' order by start_time` + ); + + await db.any(`DELETE FROM video_dialogs_ocr where video_id = '${vid_id}'`); + + for (let i = 0; i < video_dialogs.length; i++) { + let currDialog = video_dialogs[i]; + + let cd_startTime = currDialog.start_time; + let cd_endTime = currDialog.end_time; + + await db.query( + `INSERT INTO video_dialogs_ocr(video_id, start_time, end_time) VALUES('${vid_id}', ${cd_startTime}, ${cd_endTime})` + ); + } + + for (let i = 0; i < inline_times.length; i++) { + let curr = inline_times[i]; + + let it_startTime = curr.start_time; + let it_endTime = curr.end_time; + + await db.query( + `INSERT INTO video_dialogs_ocr(video_id, start_time, end_time) VALUES('${vid_id}', ${it_startTime}, ${it_endTime})` + ); + } +} +/** Add new dialog times in array as well and sort them */ +async function fetchNearestDialogInfo( + timestamp, + duration, + startTimesArr, + endTimesArr, + videoEndTime, + sceneEndTime1, + sceneEndTime2 +) { + let i = 0, + l = startTimesArr.length; + let result = {}; + + while (i < l && timestamp > startTimesArr[i]) { + i++; + } + // ts: 12 - 8s, ts: 20 - 6s + // 1 -5, 7-10, 12-20, 25-29, ...... + let startTime1 = i == 0 ? 0 : startTimesArr[i - 1]; //7 + let endTime1 = i == 0 ? 0 : endTimesArr[i - 1]; // 9 + + let startTime2 = i == l ? videoEndTime : startTimesArr[i]; // 15 + let endTime2 = i == l ? videoEndTime : endTimesArr[i]; //19 + + startTime2 = Math.min(startTime2, sceneEndTime2); + endTime1 = Math.max(endTime1, sceneEndTime1); + let gap = startTime2 - endTime1; + + //TODO: + /** + // Case 1: Start playing inlin from endTime1 instead of timestmap + //Steps - 1. gap >= duration + // => Start time of audio, a. timestamp b. starttime2 - duration + // 2. gap < duration + // => Play as extended at timestamp (check timestamp & starttime2 collision) + 3. not in gap + => set a threshold of m sec + a. endTime1 - timesatmp < m + i. gap > duration, play inline + ii. else extended at endtime1 (endtime1 == sceneEndTime2 ? endtime1+1 : endTime1) + b. endTime1 - timesatmp > m + i. (optional) if timestamp - startTime1 < m, play extended at startTime1 + ii. all other cases, play as extended at timestamp + + */ + + /** Inside time gap */ + if (timestamp > endTime1 && timestamp < startTime2) { + //1. gap >= duration + if (duration < gap) { + // a. timestamp + if (timestamp + duration < startTime2) { + result["isInline"] = true; + result["start_time"] = timestamp; + return result; + } + // b. + else { + result["isInline"] = true; + result["start_time"] = startTime2 - duration - 0.1; + return result; + } + } + //2. gap < duration + else { + // (check timestamp & starttime2 collision) + if (startTime2 == sceneEndTime2 && startTime2 - timestamp < 1) + timestamp = startTime2 + 1; + + result["isInline"] = false; + result["start_time"] = timestamp; + return result; + } + } else { + /** Not in gap */ + let threshold = 3; + //a. endTime1 - timesatmp < m + if (endTime1 - timestamp < threshold) { + // i. gap > duration, play inline + if (gap > duration) { + result["isInline"] = true; + result["start_time"] = endTime1; // or scene_end time + return result; + } + // ii. else extended at endtime1 + else { + // (endtime1 == sceneEndTime2 ? endtime1+1 : endTime1) + endTime1 = endTime1 == sceneEndTime2 ? endTime1 + 1 : endTime1; + result["isInline"] = false; + result["start_time"] = endTime1; // or scene_end time + return result; + } + } else { + result["isInline"] = false; + result["start_time"] = timestamp; // or scene_end time + return result; + } + } +} + +async function fetchVideoStartTime(vid_id, vol_id) { + let videoStartTimesArr = await db.query( + `SELECT start_time FROM descriptions WHERE video_id = '${vid_id}' and volunteer_id = '${vol_id}' ORDER BY scene_num` + ); + return videoStartTimesArr[0].start_time; + // let videoStartTimes = []; + // for(let i=0; i ref: ',reference,' and sen: ',sentence); +let dataToSend; + // spawn new child process to call the python script with parameters +const python = spawn('python', ['../python/helloworld.py',reference,sentence]); + // collect data from script + python.stdout.on('data', function (data) { + console.log('Pipe data from python script ...'); + dataToSend = data.toString(); + }); + + python.on('close', (code) => { + console.log(`child process close all stdio with code ${code}`); + return dataToSend + }); +} + +module.exports = { + generateBLEUScore: generateBLEUScore, +} \ No newline at end of file diff --git a/backend/python/bleusc.py b/backend/python/bleusc.py new file mode 100644 index 0000000..7f20b39 --- /dev/null +++ b/backend/python/bleusc.py @@ -0,0 +1,13 @@ +from nltk.translate.bleu_score import sentence_bleu +import sys + +print('#Hello from python#') + +sentence1 = sys.argv[1] +sentence2 = sys.argv[2] +sentence1 = sentence1.split(' ') +sentence2 = sentence2.split(' ') + +reference = [sentence1] +score = sentence_bleu(reference, sentence2) +print('## The BLEU score is: ',score) diff --git a/backend/sentences.js b/backend/sentences.js new file mode 100644 index 0000000..8724d9d --- /dev/null +++ b/backend/sentences.js @@ -0,0 +1,769 @@ +const textToSpeech = require('@google-cloud/text-to-speech'); +const fs = require('fs'); +const util = require('util'); +const db = require('../db/database'); +const fse = require('fs-extra'); +const { getAudioDurationInSeconds } = require('get-audio-duration') +let MediaSplit = require('media-split'); +// const detectCharacterEncoding = require('detect-character-encoding'); + + +async function mp3Split(file, midTime, callbackURL){ + + try{ + // const fileBuffer = fs.readFileSync('file.txt'); + // const charsetMatch = detectCharacterEncoding(file); + + // console.log(charsetMatch); + + + const writeFile = util.promisify(fs.writeFile); + await writeFile('public/audio/output.mp3', file, 'binary'); + console.log('Audio content written to file: output.mp3'); + console.log("time: "+midTime); + + let split = new MediaSplit({ input: "public/audio/output.mp3", sections: [`[00:00 - ${midTime}] output1`] }); + + split.parse().then((sections) => { + for (let section of sections) { + console.log(section.name); // filename + console.log(section.start); // section start + console.log(section.end); // section end + console.log(section.trackName); // track name + } + }); + + + split = new MediaSplit({ input: "public/audio/output.mp3", sections: [`[${midTime}] output2`] }); + + split.parse().then((sections) => { + for (let section of sections) { + console.log(section.name); // filename + console.log(section.start); // section start + console.log(section.end); // section end + console.log(section.trackName); // track name + } + }); + } + catch (error) { + console.log(error); + return false; + } +} + +async function refereshAudioData(vid_id, vol_id){ + + await db.any(`Delete from descriptions_sentence where video_id = '${vid_id}' AND volunteer_id = ${vol_id}`); + await db.any(`Delete from descriptions_new_sentence where video_id = '${vid_id}' AND volunteer_id = ${vol_id}`); + +} + + /** Generates mp3 file based on text. Audio is stored in output.mp3 */ +async function generateMp3ForText(inputText) { + try { + const client = new textToSpeech.TextToSpeechClient(); + const request = { + input: {text: inputText}, + voice: {languageCode: 'en-US', ssmlGender: 'NEUTRAL'}, + audioConfig: {audioEncoding: 'MP3'}, + }; + const [response] = await client.synthesizeSpeech(request); + const writeFile = util.promisify(fs.writeFile); + await writeFile('public/audio/output.mp3', response.audioContent, 'binary'); + console.log('Audio content written to file: output.mp3'); + return true; + } catch (error) { + console.log(error); + return false; + } +} + +async function generateMp3ForTextInFileystem(fileNamePath, inputText) { + try { + const client = new textToSpeech.TextToSpeechClient(); + const request = { + input: {text: inputText}, + voice: {languageCode: 'en-US', ssmlGender: 'NEUTRAL'}, + audioConfig: {audioEncoding: 'MP3'}, + }; + const [response] = await client.synthesizeSpeech(request); + const writeFile = util.promisify(fse.outputFile); + await writeFile(fileNamePath, response.audioContent, 'binary'); + console.log(`Audio content written to file: ${fileNamePath}`); + return true; + } catch (error) { + console.log(error); + return false; + } +} +/** Generates mp3 audio files for all the sentences */ +async function generateMp3ForAllSentencesInDb() { + try{ + let sentenceListArr = await db.any('SELECT * FROM descriptions_sentence order by scene_num, sentence_id'); + + for(let i=0; i0) ? "\nText on screen : " + ocrText : ""; + let lastChar = descriptionText[descriptionText.length - 1]; + if(lastChar != '.' && lastChar != '!' && lastChar != '?') + descriptionText += '.'; + let finalText = descriptionText + ocrText; + let descType = hasAi ? 'ai' : 'non_ai'; + let sent = currSentence.sentence; + let str = currSentence.sentence_id; + let newstr = str.split('_'); + let sentNum = newstr[newstr.length - 1]; + let fileNamePath = `public/audio/${currVideoId}/${descType}/${volunteerId}/${sceneNum}/${sentNum}/.mp3`; + let relativePath = fileNamePath.replace('public', '..'); + + await generateMp3ForTextInFileystem(fileNamePath, sent); + let duration = await getAudioDurationInSeconds(fileNamePath); + console.log(duration); + await updateSentencesDescriptionTable(duration, currVideoId, volunteerId, hasAi, sceneNum, relativePath, str); + } + return true; + + } catch(error){ + console.log(error); + return false; + } +} + +/** Creates separate sentences of given descriptions in the db for all videos & users */ +async function generateSentencesForAllDescriptionsInDb() { + try { + let descriptionListArr = await db.any('SELECT * FROM descriptions order by id'); + for(let i=0; i0) ? "\nText on screen : " + ocrText : ""; + let lastChar = descriptionText[descriptionText.length - 1]; + if(lastChar != '.' && lastChar != '!' && lastChar != '?') + descriptionText += '.'; + let finalText = descriptionText + ocrText; + let descType = hasAi ? 'ai' : 'non_ai'; + // let fileNamePath = `public/audio/${currVideoId}/${descType}/${volunteerId}/${sceneNum}.mp3`; + // let relativePath = fileNamePath.replace('public', '..'); + sentences = finalText.replace(/([.?!])\s*(?=[A-Za-z])/g, "$1|").split("|") + let sentence_id_key = 0; + + for(s of sentences) + { + let sentence_id = currDescriptionId + '_' + sentence_id_key; + await insertDescriptionSentencesTable(currDescriptionId,currVideoId,currSceneId, + currScene_start_time,currScene_end_time,volunteerId,hasAi,descriptionText,ocrText, + sceneNum,finalText,sentence_id,s); + sentence_id_key += 1; + } + // await generateMp3ForTextInFileystem(fileNamePath, finalText); + // let duration = await getAudioDurationInSeconds(fileNamePath); + // console.log(duration); + // await updateDescriptionTable(duration, currVideoId, volunteerId, hasAi, sceneNum, relativePath); + } + return true; + } catch (error) { + console.log(error); + return false; + } +} + +async function insertDescriptionSentencesTable(currDescriptionId,currVideoId,currSceneId, + currScene_start_time,currScene_end_time,volunteerId,hasAi,descriptionText,ocrText, + sceneNum,finalText,sentence_id,s){ + try { + await db.query(`INSERT INTO descriptions_sentence(desc_id,video_id,scene_id, + start_time,end_time,volunteer_id,has_ai,description,ocr, + scene_num,sentence_id,sentence) VALUES('${currDescriptionId}','${currVideoId}','${currSceneId}', + ${currScene_start_time},${currScene_end_time},'${volunteerId}',${hasAi},'${descriptionText}','${ocrText}', + ${sceneNum},'${sentence_id}','${s}');`); + + return true; + } catch(e) { + console.log(e); + return false; + } + } + + +async function updateSentencesDescriptionTable(duration, currVideoId, volunteerId, hasAi, sceneNum, relativePath, sentence_id) { + try { + await db.query(`UPDATE descriptions_sentence SET audio_length = ${duration}, audiopath = '${relativePath}' where video_id = '${currVideoId}' and volunteer_id = '${volunteerId}' and has_ai=${hasAi} and scene_num=${sceneNum} and sentence_id='${sentence_id}'`); + return true; + } catch(e) { + console.log(e); + return false; + } +} + /** 3rd step: Insert audio descriptions inline/extended in the video for all videos*/ +async function generateTimestampsforSentencesInDb(){ + + let sentences = await db.query(`SELECT * FROM descriptions_sentence ORDER BY scene_num, sentence_id`); + let size = 0; + let sentence_duration = []; + let scene_start_time = []; + let sentence_video_id = []; + let sentence_video_end_times = []; + let sentence_id = []; + let sentence_scene = []; + + while(size=0){ + let currVideo = sentence_video_id[i]; + if(!dialog_video_set.has(currVideo)) + { i--; continue;} + // Include prevScene as well to change the scene + let currVideo_dialog = await db.query(`SELECT * FROM video_dialogs_timestamps WHERE video_id = '${currVideo}' ORDER BY start_time`); + let currDialogStartTimes = []; + let currDialogEndTimes = []; + + for(let size=0; size < currVideo_dialog.length; size++) + { + currDialogStartTimes.push(currVideo_dialog[size]['start_time']); + currDialogEndTimes.push(currVideo_dialog[size]['end_time']); + } + + let index = currDialogStartTimes.length - 1; + let video_end_time = sentence_video_end_times[i]; + let time_gap = video_end_time - currDialogEndTimes[index]; + let time_start = video_end_time - sentence_duration[i]; // (to check sentence is within the same scene) + let time_end = video_end_time; + + while(index > 0){ + if(currVideo != sentence_video_id[i]) + break; + let end_time_sceneNum = await fetchSceneNum(time_end, currVideo, sentence_id[i]); + let start_time_sceneNum = await fetchSceneNum(time_start, currVideo, sentence_id[i]); + // if time gap is between two different scenes, break the time gap into two parts for each scene + if(start_time_sceneNum != end_time_sceneNum) + { + let presentSceneNum = sentence_scene[i]; + + while(presentSceneNum == sentence_scene[i] && currVideo == sentence_video_id[i]){ + let sceneStartTime = scene_start_time[i]; + await updateSentenceStartTime(sentence_id[i], sceneStartTime, 'extended'); + i--; + } + + index--; + time_gap = currDialogStartTimes[index+1] - currDialogEndTimes[index]; + time_start = currDialogStartTimes[index+1] - sentence_duration[i]; // (to check sentence is within the same scene) + time_end = currDialogStartTimes[index+1]; + continue; + } + + if(time_gap <= sentence_duration[i]) + { + index--; + time_gap = currDialogStartTimes[index+1] - currDialogEndTimes[index]; + time_start = currDialogStartTimes[index+1] - sentence_duration[i]; // (to check sentence is within the same scene) + time_end = currDialogStartTimes[index+1]; + continue; + } + + while(start_time_sceneNum == end_time_sceneNum && time_gap > sentence_duration[i]){ + await updateSentenceStartTime(sentence_id[i], time_start, 'inline'); + time_gap -= (sentence_duration[i] + 0.1); + time_start -= (sentence_duration[i] + 0.1); + time_end -= (sentence_duration[i] + 0.1); + + start_time_sceneNum = await fetchSceneNum(time_start, currVideo, sentence_id[i]); + end_time_sceneNum = await fetchSceneNum(time_end, currVideo, sentence_id[i]); + i--; + } + + + } + + // if time gap is between two different scenes, break the time gap into two parts for each scene + if(index==0 && currVideo == sentence_video_id[i]) + { + let presentSceneNum = sentence_scene[i]; + + while(i>= 0 && presentSceneNum == sentence_scene[i]){ + let sceneStartTime = scene_start_time[i]; + await updateSentenceStartTime(sentence_id[i], sceneStartTime, 'extended'); + i--; + } + } + +} +} + /** 3rd step: Insert audio descriptions inline/extended in the video for a particular user and video*/ +async function generateTimestampsforSentencesInDbWithInfo(dbVideoId, dbVolId){ + + let sentences = await db.query(`SELECT * FROM descriptions_sentence WHERE video_id='${dbVideoId}' and volunteer_id='${dbVolId}' ORDER BY scene_num, sentence_id`); + let size = 0; + let sentence_duration = []; + let scene_start_time = []; + let sentence_video_id = []; + let sentence_video_end_times = []; + let sentence_id = []; + let sentence_scene = []; + + while(size=0){ + let currVideo = sentence_video_id[i]; + if(!dialog_video_set.has(currVideo)) + { i--; continue;} + // Include prevScene as well to change the scene + let currVideo_dialog = await db.query(`SELECT * FROM video_dialogs WHERE video_id = '${currVideo}' ORDER BY start_time`); + let currDialogStartTimes = []; + let currDialogEndTimes = []; + + for(let size=0; size < currVideo_dialog.length; size++) + { + currDialogStartTimes.push(currVideo_dialog[size]['start_time']); + currDialogEndTimes.push(currVideo_dialog[size]['end_time']); + } + + let index = currDialogStartTimes.length - 1; + let video_end_time = sentence_video_end_times[i]; + let time_gap = video_end_time - Math.max(currDialogEndTimes[index],scene_start_time[i]); + let time_start = video_end_time - sentence_duration[i]; // (to check sentence is within the same scene) + let time_end = video_end_time; + let currSceneNum = sentence_scene[i]; + + while(index >= -1){ + if(currVideo != sentence_video_id[i]) + break; + let end_time_sceneNum = await fetchSceneNum(time_end, currVideo, sentence_id[i]); + let start_time_sceneNum = await fetchSceneNum(time_start, currVideo, sentence_id[i]); + + // if time gap is between two different scenes, break the time gap into two parts for each scene + if(start_time_sceneNum != end_time_sceneNum || time_gap == 0 || end_time_sceneNum != currSceneNum) + { + let presentSceneNum = sentence_scene[i]; + + while(presentSceneNum == sentence_scene[i] && currVideo == sentence_video_id[i]){ + let sceneStartTime = scene_start_time[i]; + await updateSentenceStartTime(sentence_id[i], sceneStartTime, 'extended'); + i--; + } + + // index--; + // time_gap = currDialogStartTimes[index+1] - Math.max(currDialogEndTimes[index],scene_start_time[i]); + // time_start = currDialogStartTimes[index+1] - sentence_duration[i]; // (to check sentence is within the same scene) + // time_end = currDialogStartTimes[index+1]; + // continue; + } + + if(time_gap <= sentence_duration[i] && currSceneNum == sentence_scene[i]) + { + curr_dialog_start_time = currDialogStartTimes[index]; + present_scene_num = await fetchSceneNum(curr_dialog_start_time, currVideo, sentence_id[i]); + if(present_scene_num < currSceneNum) + {time_gap = 0; continue;} + + index--; + + let currDial_endTime = (index >= 0 ? currDialogEndTimes[index] : scene_start_time[i]); // deals with 1st scene edge case + + time_gap = currDialogStartTimes[index+1] - Math.max(currDial_endTime,scene_start_time[i]); + time_start = currDialogStartTimes[index+1] - sentence_duration[i] - 0.1; // (to check sentence is within the same scene) + time_end = currDialogStartTimes[index+1] - 0.1; + /*************** Major bug in listen by code *******************/ + if (time_gap == 0) + time_gap = 0.01; + /*************** *************************** *******************/ + continue; + } + + while(start_time_sceneNum == end_time_sceneNum && time_gap > sentence_duration[i] && currVideo == sentence_video_id[i] && currSceneNum == sentence_scene[i]){ + await updateSentenceStartTime(sentence_id[i], time_start, 'inline'); + time_gap -= (sentence_duration[i] + 0.1); + time_end = time_start - 0.1; + // time_start -= (sentence_duration[i] + 0.1); + + end_time_sceneNum = await fetchSceneNum(time_end, currVideo, sentence_id[i]); + i--; + if(i<0){index = -2; continue;} + + time_start = time_end - sentence_duration[i]; + start_time_sceneNum = await fetchSceneNum(time_start, currVideo, sentence_id[i]); + } + + if(currSceneNum != sentence_scene[i] && currVideo == sentence_video_id[i]){ + currSceneNum = sentence_scene[i]; + // handles 6 edge cases + while(true){ + + // no dialogs left + if(index==-1) + { + time_gap = currDialogStartTimes[0]; + time_end = currDialogStartTimes[0] - 0.1; + time_start = time_end - sentence_duration[i]; + break; + }else if(index < -1) + break; + + + let dialog_start_time_sceneNum = await fetchSceneNum(currDialogStartTimes[index], currVideo, sentence_id[i]); + let dialog_end_time_sceneNum = await fetchSceneNum(currDialogEndTimes[index], currVideo, sentence_id[i]); + + // if dialog's starting time jumps to previous scene + if(dialog_start_time_sceneNum < sentence_scene[i]){ + + // if dialog's ending time also jumps to previous scene + if(dialog_end_time_sceneNum < sentence_scene[i]) + { + time_gap = scene_start_time[i+1] - scene_start_time[i]; + time_end = scene_start_time[i+1] - 0.1; + time_start = time_end - sentence_duration[i]//scene_start_time[i+1]; + break; + } + + // if dialog's ending time is in current scene + if(dialog_end_time_sceneNum == sentence_scene[i]) + { + time_gap = scene_start_time[i+1] - currDialogEndTimes[index]; + time_end = scene_start_time[i+1] - 0.1; + time_start = time_end - sentence_duration[i]; + break; + } + + // if dialog's ending time is in next scene + if(dialog_end_time_sceneNum > sentence_scene[i]) + { + time_gap = 0; + time_end = scene_start_time[i+1] - 0.1; + time_start = time_end - sentence_duration[i]; + break; + } + } + + if(dialog_start_time_sceneNum == sentence_scene[i]){ + // Both dialog timestamps in the same scene + if(dialog_end_time_sceneNum == sentence_scene[i]){ + time_gap = scene_start_time[i+1] - currDialogEndTimes[index]; + time_end = scene_start_time[i+1] - 0.1; + time_start = time_end - sentence_duration[i]; + break; + } + // dialog end time is next scene (?) + else{ + time_gap = currDialogStartTimes[index] - Math.max(currDialogEndTimes[index-1],scene_start_time[i]); + time_end = currDialogStartTimes[index]; + time_start = time_end - sentence_duration[i]; + break; + } + } + + + + index--; + } + + // let time_gap = video_end_time - Math.max(currDialogEndTimes[index],scene_start_time[i]); + // let time_start = video_end_time - sentence_duration[i]; // (to check sentence is within the same scene) + // let time_end = video_end_time; + + continue; + } + else if(start_time_sceneNum != end_time_sceneNum){ + continue; + } + if (index < 0) + break; + curr_dialog_start_time = currDialogStartTimes[index]; + present_scene_num = await fetchSceneNum(curr_dialog_start_time, currVideo, sentence_id[i]); + + if(present_scene_num < currSceneNum) + {time_gap = 0; continue;} + + index--; + + time_gap = currDialogStartTimes[index+1] - Math.max(currDialogEndTimes[index],scene_start_time[i]); + time_end = currDialogStartTimes[index]; + time_start = time_end - sentence_duration[i]; + + // time_gap = video_end_time - Math.max(currDialogEndTimes[index],scene_start_time[i]); + // time_start = video_end_time - sentence_duration[i]; // (to check sentence is within the same scene) + // time_end = video_end_time; + + } + + // if time gap is between two different scenes, break the time gap into two parts for each scene + if(index==0 && currVideo == sentence_video_id[i]) + { + let presentSceneNum = sentence_scene[i]; + + while(i>= 0 && presentSceneNum == sentence_scene[i]){ + let sceneStartTime = scene_start_time[i]; + await updateSentenceStartTime(sentence_id[i], sceneStartTime, 'extended'); + i--; + } + } + +} +} + /** Fetch scene num from a given timestamp, videoId */ +async function fetchSceneNum(timestamp, videoId, sentence_id){ + + console.log('Scene timestamps: '+timestamp+' and '+videoId+' and '+sentence_id); + let newstr = sentence_id.split('_'); + let volunteerId = newstr[0]; + + let sceneStartTimesArr = await db.query(`SELECT start_time FROM descriptions_sentence WHERE video_id = '${videoId}' AND volunteer_id = '${volunteerId}' ORDER BY scene_num`); + let sceneNumArr = await db.query(`SELECT scene_num FROM descriptions_sentence WHERE video_id = '${videoId}' AND volunteer_id = '${volunteerId}' ORDER BY scene_num`); + // console.log('Scene timestamps: '+sceneStartTimesArr+' and '+sceneNumArr); + if(timestamp < 0) + return -1; + + for(let i=0; i timestamp) + return sceneNumArr[i-1]['scene_num']; + } + + return sceneNumArr[sceneNumArr.length - 1]['scene_num']; +} + +async function updateSentenceStartTime(sentence_id, audioStartTime, audioType){ + console.log('Updating sentence timestamp of ' + sentence_id+' : '+audioStartTime+' type: '+audioType); + try { + await db.query(`UPDATE descriptions_sentence SET audio_start_time = ${audioStartTime}, audio_type = '${audioType}' where sentence_id = '${sentence_id}'`); + return true; + } catch(e) { + console.log(e); + return false; + } + +} + +async function mergeSentenceAudios(video_id, vol_id){ + let sentenceListArr = await db.any(`SELECT * FROM descriptions_sentence WHERE video_id = '${video_id}' AND volunteer_id = '${vol_id}' ORDER BY scene_num, sentence_id`); + let size = 0; + let sentence = []; + let sentence_id = []; + let sentence_desc = []; + let sentence_ocr = []; + let sentence_duration = []; + let scene_start_time = []; + let scene_end_time = []; + let sentence_video_id = []; + let sentence_video_end_times = []; + let sentence_scene = []; + let sentence_audio_type = []; + let sentence_audio_start_time = []; + let sentence_audio_path = []; + + while(size setTimeout(r, 100)); + console.log(`New time: ` + date.getTime()); + return true; + } catch (error) { + console.log(error); + return false; + } } async function generateMp3ForAllDescriptionsInDb() { - try { - let descriptionListArr = await db.any('SELECT * FROM descriptions order by id'); - for(let i=0; i0) ? "\nText on screen is : " + ocrText : ""; - let finalText = descriptionText + ocrText; - let descType = hasAi ? 'ai' : 'non_ai'; - let fileNamePath = `public/audio/${currVideoId}/${descType}/${volunteerId}/${sceneNum}.mp3`; - let relativePath = fileNamePath.replace('public', '..'); - await generateMp3ForTextInFileystem(fileNamePath, finalText); - let duration = await getAudioDurationInSeconds(fileNamePath); - console.log(duration); - await updateDescriptionTable(duration, currVideoId, volunteerId, hasAi, sceneNum, relativePath); - } - return true; - } catch (error) { - console.log(error); - return false; + try { + let descriptionListArr = await db.any( + "SELECT * FROM descriptions order by id" + ); + for (let i = 0; i < descriptionListArr.length; i++) { + let currDescription = descriptionListArr[i]; + let currDescriptionId = currDescription.desc_id; + let currVideoId = currDescription.video_id; + let volunteerId = currDescription.volunteer_id; + let hasAi = currDescription.has_ai; + let descriptionText = currDescription.description; + let ocrText = currDescription.ocr; + let sceneNum = currDescription.scene_num; + ocrText = + hasAi && ocrText && ocrText.trim().length > 0 + ? "\nText on screen : " + ocrText + : ""; + let lastChar = descriptionText[descriptionText.length - 1]; + if (lastChar != "." && lastChar != "!" && lastChar != "?") + descriptionText += "."; + let finalText = descriptionText + ocrText; + let descType = hasAi ? "ai" : "non_ai"; + let fileNamePath = `public/audio/${currVideoId}/${descType}/${volunteerId}/${sceneNum}.mp3`; + let relativePath = fileNamePath.replace("public", ".."); + await generateMp3ForTextInFileystem(fileNamePath, finalText); + let duration = await getAudioDurationInSeconds(fileNamePath); + console.log(duration); + await updateDescriptionTable( + duration, + currVideoId, + volunteerId, + hasAi, + sceneNum, + relativePath + ); } + return true; + } catch (error) { + console.log(error); + return false; + } } -async function updateDescriptionTable(duration, currVideoId, volunteerId, hasAi, sceneNum, relativePath) { - try { - await db.query(`UPDATE descriptions SET audio_length = ${duration}, audiopath = '${relativePath}' where video_id = '${currVideoId}' and volunteer_id = '${volunteerId}' and has_ai=${hasAi} and scene_num=${sceneNum}`); - return true; - } catch(e) { - console.log(e); - return false; - } +async function updateDescriptionTable( + duration, + currVideoId, + volunteerId, + hasAi, + sceneNum, + relativePath +) { + try { + await db.query( + `UPDATE descriptions SET audio_length = ${duration}, audiopath = '${relativePath}' where video_id = '${currVideoId}' and volunteer_id = '${volunteerId}' and has_ai=${hasAi} and scene_num=${sceneNum}` + ); + return true; + } catch (e) { + console.log(e); + return false; + } +} + +async function getMp3ForDescription(inputText, res) { + try { + const client = new textToSpeech.TextToSpeechClient(); + const request = { + input: { text: inputText }, + voice: { languageCode: "en-US", ssmlGender: "NEUTRAL" }, + audioConfig: { audioEncoding: "MP3" } + }; + const [response] = await client.synthesizeSpeech(request); + const writeFile = util.promisify(fs.writeFile); + const fileNamePath = "public/audio/output.mp3"; + await writeFile(fileNamePath, response.audioContent, "binary"); + console.log("Audio content written to file: output.mp3"); + + // let form = new FormData(); + + let readStream = fs.createReadStream("./" + fileNamePath); + readStream.on("error", function(err) { + console.log("RS err is: ", err); + reject(err); + }); + + res.set("content-type", "audio/mp3"); + res.set("content-disposition", "output.mp3"); + res.set("accept-ranges", "bytes"); + + readStream.on("data", function(chunk) { + console.log("File opened"); + // reject(err) + res.write(chunk); + }); + + readStream.on("end", () => { + res.end(); + }); + + console.log("RS err"); + const formData = { + file: readStream, + callbackurl: "" + }; + + let duration = await getAudioDurationInSeconds(fileNamePath); + + return { + success: true, + duration: duration, + file: response, + form: formData + }; + } catch (error) { + console.log(error); + return { success: false, duration: "None" }; + } +} + +// Added file in response +async function generateMp3DurationForText(inputText) { + try { + const client = new textToSpeech.TextToSpeechClient(); + const request = { + input: { text: inputText }, + voice: { languageCode: "en-US", ssmlGender: "NEUTRAL" }, + audioConfig: { audioEncoding: "MP3" } + }; + const [response] = await client.synthesizeSpeech(request); + const writeFile = util.promisify(fs.writeFile); + const fileNamePath = "public/audio/output.mp3"; + await writeFile(fileNamePath, response.audioContent, "binary"); + console.log("Audio content written to file: output.mp3"); + + let readStream = fs.createReadStream("./" + fileNamePath); + readStream.on("error", function(err) { + console.log("RS err is: ", err); + // reject(err); + }); + + readStream.on("data", function(chunk) { + console.log("File opened"); + // reject(err) + }); + + readStream.on("end", () => {}); + + console.log("RS err"); + const formData = { + file: readStream, + callbackurl: "" + }; + + let duration = await getAudioDurationInSeconds(fileNamePath); + + return { + success: true, + duration: duration, + file: response, + form: formData + }; + } catch (error) { + console.log(error); + return { success: false, duration: "None" }; + } } module.exports = { - generateMp3ForText: generateMp3ForText, - generateMp3ForAllDescriptionsInDb: generateMp3ForAllDescriptionsInDb -} \ No newline at end of file + getMp3ForDescription: getMp3ForDescription, + generateMp3ForText: generateMp3ForText, + generateMp3ForTextInFileystem: generateMp3ForTextInFileystem, + generateMp3ForAllDescriptionsInDb: generateMp3ForAllDescriptionsInDb, + generateMp3DurationForText: generateMp3DurationForText +}; diff --git a/backend/video.js b/backend/video.js index ec08d18..059b863 100644 --- a/backend/video.js +++ b/backend/video.js @@ -57,6 +57,84 @@ async function fetchAllVideos() { return await db.any('SELECT id, youtube_id, title, duration FROM video'); } +async function fetchScene(videoId) { + return await db.query('SELECT * FROM scene where video_id=$1 order by scene_num', [videoId]); +} + +async function fetchAllKeyFrames(videoId) { + + let match = videoId; //sceneId.split('_')[0]; + console.log('match: ',match); + match += '%'; + console.log('match: ',match); + result = await db.any(`SELECT thumbnail_url, caption, pythia_caption from keyframe where keyframe_id::text LIKE '${match}'`); + return result; +} + + /** Fetch last 5 keyframes irrespective of scene numbers */ +async function fetchKeyFrames(videoId,videoTime) { + + // result = await db.any(`SELECT keyframe_url, pythia_caption, keyframe_num from keyframe_revised where keyframe_id::text LIKE '${match}' AND timestamp < ${videoTime} ORDER BY timestamp DESC LIMIT 1`); + result = await db.any(`SELECT keyframe_url, pythia_caption, keyframe_num from keyframe_revised where video_id = '${videoId}' AND timestamp < ${videoTime} ORDER BY timestamp DESC LIMIT 1`); + console.log('res: ',result); + return result.reverse(); +} + +async function fetchRevisedKeyFrames(vidId, videoTime){ + if (videoTime==null){ + result = await db.any(`SELECT * from keyframe_revised where video_id = '${vidId}' ORDER BY timestamp`); + console.log('res: ',result); + return result; + } + return []; +} + +async function fetchDialogTimeStamps(videoId) { + console.log("Ent: testking"); + return await db.any(`SELECT start_time, end_time FROM video_dialogs_timestamps WHERE video_id = '${videoId}' ORDER BY start_time`); + // return await db.any("SELECT shot_id FROM shot where scene_id = 'nqXz8hhAYGo_3_5' "); +} + +async function fetchVideoData(videoId, userId) { + + try { + let vidQueryResult = await db.one('SELECT youtube_id, indexer_video_id, title, duration FROM video WHERE youtube_id = $1', [videoId]); + + let queryResult = await db.query(`select scene_id as sc_id, scene_num as sc_num, start_time, end_time, + original_ocr, pythia_caption_bleu as original_description from scene where video_id= $1 ORDER BY scene_num`, [videoId]); + queryResult.forEach(scene => { + scene.deletedScene = []; + scene.note = ''; + }); + + let mergeSceneResult = await db.query('SELECT scene_id as sc_id, merged FROM merge_scene WHERE video_id = $1 order by scene_num', [videoId]); + + let modifiedData = await db.query(`select scene_id as sc_id, scene_num as sc_num, + start_time, end_time, ocr as original_ocr, + description as original_description, merged_scene as "deletedScene", note + from descriptions where volunteer_id = $1 and + has_ai=true and + video_id= $2 ORDER BY scene_num` + , [userId , videoId]) + // console.log('over data', JSON.stringify(modifiedData)); + vidQueryResult.originalData = queryResult; + vidQueryResult.modifiedData = modifiedData; + vidQueryResult.mergeSceneRecommendation = mergeSceneResult; + console.log({vidQueryResult}); + // console.log('vidQueryResult ===>>', JSON.stringify(vidQueryResult)); + return vidQueryResult; + } catch(e) { + throw e; + } +} + +async function fetchSentences(videoId, userId) { + console.log("Now: testking"); + return await db.any(`SELECT * FROM descriptions_new_sentence WHERE video_id = '${videoId}' AND volunteer_id = '${userId}' ORDER BY scene_num, audio_start_time`); + // return await db.any("SELECT shot_id FROM shot where scene_id = 'nqXz8hhAYGo_3_5' "); +} + + async function fetchVideoDataFromDb(videoId, userId) { try { let vidQueryResult = await db.one('SELECT youtube_id, indexer_video_id, title, duration FROM video WHERE youtube_id = $1', [videoId]); @@ -65,11 +143,12 @@ async function fetchVideoDataFromDb(videoId, userId) { 'block_num', block_num, 'start_time', start_time, 'finish_time', end_time, 'baseline_des', original_description, 'child_scenes', jsonb_agg(scene) ) js_object FROM ( select b.*, jsonb_build_object( 'sc_id', s.scene_id, 'sc_num', - s.scene_num, 'start_time', s.start_time, 'finish_time', s.end_time, 'original_description', s.original_description, + s.scene_num, 'start_time', s.start_time, 'finish_time', s.end_time, 'original_description', s.pythia_caption, 'original_ocr', s.original_ocr ) scene FROM block b JOIN scene s ON s.block_id = b.block_id AND b.video_id = $1 ORDER BY scene_num ) s GROUP BY start_time, end_time, block_id, original_description, block_num, video_id ORDER BY block_num ASC ) s`, [videoId]); + let overrideData = await db.query(`select scene_id as sc_id, scene_num as sc_num, start_time, end_time as finish_time, ocr as original_ocr, description as original_description, merged_scene as "deletedScene" @@ -164,8 +243,22 @@ async function replicateSceneDataInModifiedSceneTable(userId, videoId) { } } + +function filterOCR(ocrArr, duration, confidenceScore, topLimit){ + for (const ocrObj of ocrArr) { + let diff = ocrObj.end_time - ocrObj.start_time; + if(diff >= duration && ocrObj.confidence >= confidenceScore && ocrObj.top >= topLimit){ + + }else{ + ocrObj.text = ''; + } + } + return ocrArr; +} + //Parse and insert all JSON data from the videoindexer into the database async function addVideoDataToDb(inputJson) { + // console.log({addVideoDataToDb: inputJson}); try { // //Parse and insert Video let video = Video.parseVideoFromIndexerJson(inputJson); @@ -195,7 +288,7 @@ async function addVideoDataToDb(inputJson) { // await Block.insertIntoBlockTable(blockArr); //Parse Scenes let sceneJsonArr = video.insights_json.scenes; - let sceneArr = Scene.parseSceneArr(sceneJsonArr); + let sceneArr = Scene.parseSceneArr(sceneJsonArr, video.youtube_id); //Generate blocks from scenes using logic - TODO: change this //let blockArr = generateBlocksFromScenes(sceneArr, video.youtube_id); //await Block.insertIntoBlockTable(blockArr); @@ -217,35 +310,62 @@ async function addVideoDataToDb(inputJson) { await Keyframe.insertIntoKeyframeTable(allKeyframesArr); //Parse, Sort, Process and insert OCR data into the shot table let ocrJsonArr = video.insights_json.ocr; - let ocrArr = Ocr.parseOcrJsonArr(ocrJsonArr); - ocrArr.sort((ocr1, ocr2) => (ocr1.start_time > ocr2.start_time) ? 1 : (ocr1.start_time == ocr2.start_time) ? ((ocr1.top > ocr2.top) ? 1 : (ocr1.top == ocr2.top) ? ((ocr1.left > ocr2.left) ? 1 : -1) : -1 ): -1); - ocrArr = setShotIdForAllOcrDetections(ocrArr, shotArr); - ocrArr = cleanUpOcrDataWithinEachShot(ocrArr); - let concatenatedOcrTextArr = concatOcrWithinSameShot(ocrArr); - await Ocr.updateShotTableWithOcrData(concatenatedOcrTextArr); - //Aggregate OCR data from the Shot table and insert it into the scene table - for(let i=0; i (ocr1.start_time > ocr2.start_time) ? 1 : (ocr1.start_time == ocr2.start_time) ? ((ocr1.top > ocr2.top) ? 1 : (ocr1.top == ocr2.top) ? ((ocr1.left > ocr2.left) ? 1 : -1) : -1 ): -1); + ocrArr = setShotIdForAllOcrDetections(ocrArr, shotArr); + ocrArr = cleanUpOcrDataWithinEachShot(ocrArr); + //--------- + // filter_OCRArr = filterOCR(ocrArr, 1, 0.56, 50); + // let concatFilterOcrArr = concatOcrWithinSameShot(filter_OCRArr); + // await Ocr.updateShotTableWithFilteredOcr(concatFilterOcrArr); + //----- + let concatenatedOcrTextArr = concatOcrWithinSameShot(ocrArr); + await Ocr.updateShotTableWithOcrData(concatenatedOcrTextArr); + console.log({concatenatedOcrTextArr}); + //Aggregate OCR data from the Shot table and insert it into the scene table + for(let i=0; i + a.id); + // await db.one("UPDATE scene SET original_ocr = $1, filtered_ocr = $2 WHERE scene_id = $3 RETURNING id", [finalSceneOcr, finalFilteredSceneOcr, currSceneId], a => + a.id); + + }catch(e) { + console.log(e); } - finalSceneOcr = finalSceneOcr.trim(); - await db.one("UPDATE scene SET original_ocr = $1 WHERE scene_id = $2 RETURNING id", [finalSceneOcr, currSceneId], a => + a.id); - }catch(e) { - console.log(e); } } + //await Scene.aggregateShotOcrIntoSceneOcr(video); //Fetch captions for all keyframes and update the keyframes table @@ -281,7 +401,7 @@ async function addVideoDataToDb(inputJson) { return true; } catch(e) { console.log(e); - return false; + return e; } } @@ -554,12 +674,19 @@ async function clearDb() { module.exports = { addVideoDataToDb: addVideoDataToDb, clearDb: clearDb, + fetchDialogTimeStamps: fetchDialogTimeStamps, + fetchSentences: fetchSentences, fetchVideoDataFromDb: fetchVideoDataFromDb, fetchAllVideos: fetchAllVideos, + fetchKeyFrames: fetchKeyFrames, + fetchAllKeyFrames: fetchAllKeyFrames, updateBlockDescription: updateBlockDescription, updateSceneDescription: updateSceneDescription, updateSceneOcr: updateSceneOcr, replicateSceneDataInModifiedSceneTable: replicateSceneDataInModifiedSceneTable, updateMultipleScenesAndBlocksForUser: updateMultipleScenesAndBlocksForUser, - updateBlockAndScene: updateBlockAndScene + updateBlockAndScene: updateBlockAndScene, + fetchVideoData: fetchVideoData, + fetchScene: fetchScene, + fetchRevisedKeyFrames: fetchRevisedKeyFrames } diff --git a/bin/www b/bin/www index 9605aab..5c6aac1 100755 --- a/bin/www +++ b/bin/www @@ -7,20 +7,32 @@ var app = require('../app'); var debug = require('debug')('usr-yd-prototype:server'); var http = require('http'); +const https = require('https'); +const fs = require('fs'); +const httpsOptions = { + key: fs.readFileSync('key.pem'), + cert: fs.readFileSync('cert.pem') +}; +var express = require('express'); /** * Get port from environment and store in Express. */ - -var port = normalizePort(process.env.PORT || '5001'); +var port = normalizePort(process.env.PORT || 5001); app.set('port', port); - /** - * Create HTTP server. + * Listen on provided port, on all network interfaces. */ +// var server = https.createServer(httpsOptions, app); var server = http.createServer(app); +server.listen(port); +server.setTimeout(1*60*60*1000); +server.on('error', onError); +server.on('listening', onListening); + const io = require('socket.io')(server); + let rooms = { }; rooms['prepSession1'] = { users: {} }; @@ -29,7 +41,7 @@ rooms['session3'] = { users: {} }; rooms['session3'] = { users: {} }; rooms['1'] = { users: {} }; rooms['2'] = { users: {} }; -console.log({rooms}); +// console.log({rooms}); //chat message is an event name io.on('connection', socket =>{ @@ -38,15 +50,15 @@ io.on('connection', socket =>{ socket.on('new-user', (room, name) => { socket.join(room); // users[socket.id] = name; - console.log('HERE:', rooms); + // console.log('HERE:', rooms); rooms[room].users[socket.id] = name; socket.to(room).broadcast.emit('user-connected', name); - console.log('new user', name); - console.log('new user', rooms); + // console.log('new user', name); + // console.log('new user', rooms); }) socket.on('send-chat-message', (room, message, questionId, userId, type) => { - console.log('INside send:', room); + // console.log('INside send:', room); let data = {name: rooms[room].users[socket.id], message, questionId, @@ -73,14 +85,7 @@ function getUserRooms(socket) { return names }, []) } -/** - * Listen on provided port, on all network interfaces. - */ -server.listen(port); -server.setTimeout(1*60*60*1000); -server.on('error', onError); -server.on('listening', onListening); /** * Normalize a port into a number, string, or false. @@ -140,4 +145,6 @@ function onListening() { ? 'pipe ' + addr : 'port ' + addr.port; debug('Listening on ' + bind); + console.log(`App listening on port: ${port}`); + } diff --git a/cert.pem b/cert.pem new file mode 100644 index 0000000..c38cc1d --- /dev/null +++ b/cert.pem @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICyzCCAbMCFCKlMlU1igU6o9pZVlL8IoS5JNkRMA0GCSqGSIb3DQEBCwUAMCIx +CzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMB4XDTIwMDQwNTIyNTQw +MFoXDTQ3MDgyMTIyNTQwMFowIjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlm +b3JuaWEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDGPxgwxdq8KSWN +JALRdnyfdKlD3Pt5MxVBZ6onnkiz6ijsf45Pvfjp1nJQiZIu0ytZGMkUxh8t9PwM +K/2lBWy3KSUJRgQq9/FEknjr6YLl1GaS3Uux+2Nd3IOvykuF4FfoIizmRdADohIh +SGqhsBecGcudDWGgc6hQFrkhhTs48sRx6OyoeNdAsSt88kjzSkdaKzQxA4uc795v +eJT+bHVrB5snK9TDBZzZ0mmJUn/ilCyu53A64lZqN04eXcyJ6lzBJJv0yG6jQpgI +2LKKE95Lp/PLMKCqCSa19TWo9IbkdZuw1nFX03RprH7RxcvzFL77rhEMXrM5jf0r +ZQoZ650pAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAFRFf2TXI3iobJI8ePkc4Idd +fYil7o6VRUe1jzZGJ3d9VpiIbo8CghZCS/d1e2Quoc4mGRvD+abT+p9OhHjrGRqu +KB1XrAqmgsCLVHYAnvzgJi+9wzhUL4lUI0n3TL7K147enFCWhhiyRNYHyyFE+iuV +0rqyFtwCjJIfp5rk0fxUAQ6lBpT8NO3l4hbG3hCgvJ4/mhDxckPy43BZFRmeUkli +SKkwdlz24t29AsxF1eou2O+deL5Xa4D5HX8LBnitfnMFlyNonlx377jId+USXsSO +9e/eVJpPU/E7ryklMkYaQT7gajG6gZg8J8ZKs+KsvL2QjKFSYqw9rtPdElrHS5Y= +-----END CERTIFICATE----- diff --git a/controllers/FilterBaselineDescriptionController.js b/controllers/FilterBaselineDescriptionController.js new file mode 100644 index 0000000..86c7976 --- /dev/null +++ b/controllers/FilterBaselineDescriptionController.js @@ -0,0 +1,125 @@ +const db = require('../db/database'); +let videoModel = require('../model/video.js'); +const axios = require('axios'); +const sceneModel = require('../model/scene'); + + +class FilterBaselineDescriptionController{ + + //filter original description from Video Insight + static async filter(videoId){ + try{ + let sceneData = await sceneModel.getSceneData(videoId); + for(let i=0; i < sceneData.length; i++){ + let scene = sceneData[i]; + let desc = await this.calculate(scene.original_description); + scene.description_bleu = desc; + console.log({scene}); + } + + // update Bleu description in scene table + let res = await sceneModel.updateBleuDescription_Scene(sceneData); + console.log({res}); + return res; + }catch(e){ + console.log(e); + throw e; + } + + } + + + //Filter Pythia caption + static async filterPythiaCaption(videoId){ + try{ + let sceneData = await sceneModel.getSceneData(videoId); + for(let i=0; i < sceneData.length; i++){ + let scene = sceneData[i]; + let desc = await this.calculate(scene.pythia_caption); + scene.pythia_caption_bleu = desc; + console.log({scene}); + } + // update Bleu description in scene table + let res = await sceneModel.updateBleuPythiaCaption_Scene(sceneData); + console.log({res}); + return res; + }catch(e){ + console.log(e); + throw e; + } + } + + + //MERGE_SCENE + static async filter_MergeSceneTable(){ + try{ + let sceneData = await sceneModel.getSceneDataFromMerge(videoId); + for(let i=0; i < sceneData.length; i++){ + let scene = sceneData[i]; + let desc = await this.calculate(scene.original_description); + scene.description_bleu = desc; + console.log({scene}); + } + + let res = await sceneModel.updateBleuDescription_MergeScene(sceneData); + console.log({res}); + return res; + }catch(e){ + console.log(e); + throw e; + } + } + + static async scoreForDescription(){ + let desc = `Brian Dietzen smiling for the camera\na person sitting on a bench\nBrian Dietzen smiling for the camera + Erinn Hayes smiling for the camera\nBrian Dietzen looking at the camera + Erinn Hayes smiling for the camera\nBrian Dietzen posing for the camera`; + + let res = await this.calculate(desc); + console.log({res}); + } + + + static async calculate(description){ + let result = []; + let rejected = []; + let queue = description.split('\n'); + if(queue.length == 1){ + return queue[0]; + } + + while(queue.length > 1){ + let sentence = queue.shift(); + let body = { + sentence, + reference: queue + } + try { + let res = await axios.post('http://localhost:3005/score', body); + let score = res.data; + if(score < 0.56){ + result.push({sentence, score}); + }else{ + rejected.push({sentence , score}); + } + // console.log({rejected}); + // console.log({result}); + }catch(e){ + console.log(e); + } + } + result.push({sentence: queue[0]}); + // console.log({rejected}); + // console.log({result}); + + let desc = ''; + result.forEach((element)=>{ + desc = desc + element.sentence + '\n'; + }); + + return desc; + } + +} + +module.exports = FilterBaselineDescriptionController; \ No newline at end of file diff --git a/controllers/MergeSceneController.js b/controllers/MergeSceneController.js new file mode 100644 index 0000000..8b9a89e --- /dev/null +++ b/controllers/MergeSceneController.js @@ -0,0 +1,168 @@ + +const fs = require('fs'); +const Utils = require('../backend/utils'); +const axios = require('axios'); +const scene = require('../model/scene'); + +class MergeSceneController{ + + + execute(videoId){ + let queryResult = scene.getSceneData(videoId); + queryResult + .then(data => { + console.log({data}); + let videoInsight = []; + for(let i=0; i { + // res.send("Data added successfully in merge_scene table"); + console.log({success: data}); + }) + .catch((err) => { + console.log(err); + }); + + }) + .catch((err) => { + console.log(err); + }); + } + + + /* + execute(video_id){ + let oceanFile = 'audio_ocean8.json'; + let audioInsight = this.parseAudioFile(oceanFile); + console.log({audioInsight}); + + let url = '/merge/sceneData?videoId=' + video_id; + + axios.get(url) + .then(function (data) { + console.log(data); + + + let videoInsight = []; + for(let i=0; i= s1 && s2 <= e1) && (e2 >= s1 && e2 <= e1)) { + i++; + continue; + } + + if(Math.min(s2, e2) >= e1) { + mergedList.push(videoInsight[videoIndex++]); + continue; + } + + //merge condition + if(s2 >= s1 && e2 > e1) { + let sceneObj = mergedList[mergedList.length-1]; + let next = videoInsight[videoIndex++]; + let min = Math.min(s1, next.start_time); + let max = Math.max(e1, next.end_time); + sceneObj.start_time = min; + sceneObj.end_time = max; + // let arr = sceneObj.merged; + // arr.push(next.scene_id); + sceneObj.merged.push(next.scene_id); + mergedList.splice(mergedList.length-1); + mergedList.push(sceneObj); + }else if(s2 >= e1) { + mergedList.push(videoInsight[videoIndex++]); + } + + i++; + } + + for(let j = videoIndex; j (ocr1.start_time > ocr2.start_time) ? 1 : (ocr1.start_time == ocr2.start_time) ? ((ocr1.top > ocr2.top) ? 1 : (ocr1.top == ocr2.top) ? ((ocr1.left > ocr2.left) ? 1 : -1) : -1 ): -1); + + ocrArr = this.setShotIdForAllOcrDetections(ocrArr, shotArr); + // console.log({ocrArr}); + ocrArr = this.cleanUpOcrDataWithinEachShot(ocrArr); + for(let i of ocrArr){ + + let diff = i.end_time - i.start_time; + let obj = { + start_time: i.start_time, + end_time: i.end_time, + diff + } + // console.log({obj}); + // console.log(i.confidence, 'text: ', i.text, ' top: ',i.top, 'left: ', i.left, ' diff:', diff); + } + // console.log({ocrArr}); + let filteredOcrArr = this.filer(ocrArr, duration, confidenceScore, topLimit); + let concatenatedFilteredOCRArr = this.concatOcrWithinSameShot(filteredOcrArr); + + let concatenatedOcrTextArr = this.concatOcrWithinSameShot(ocrArr); + + let res = await OcrModel.updateShotTableWithFilteredOcr(concatenatedFilteredOCRArr); + // console.log({concatenatedFilteredOCRArr}); + // let res = await this.updateTestShotTable(concatenatedOcrTextArr); + console.log('Success: Data updated', res); + + //SCENE + let sceneArr = await db.any('SELECT * FROM scene WHERE video_id like $1 ORDER BY start_time', [videoId]); + console.log({sceneArr}); + for(let i=0; i + a.id); + + // let sceneRes = await db.one("UPDATE scene SET ocr = $1, filtered_ocr = $2 WHERE scene_id = $3 RETURNING id", [finalSceneOcr, finalFilteredSceneOcr, currSceneId], a => + a.id); + let sceneRes = await db.one("UPDATE scene SET filtered_ocr = $1 WHERE scene_id = $2 RETURNING id", [finalFilteredSceneOcr, currSceneId], a => + a.id); + + console.log('Success: Data updated', sceneRes); + } + + + + + }catch(e){ + console.log(e); + } + + + } + + + filer(ocrArr, duration, confidenceScore, topLimit){ + for (const ocrObj of ocrArr) { + let diff = ocrObj.end_time - ocrObj.start_time; + if(diff >= duration && ocrObj.confidence >= confidenceScore && ocrObj.top >= topLimit){ + + }else{ + ocrObj.text = ''; + } + } + + return ocrArr; + + } + + + async updateTestShotTable(concatenatedOcrTextArr) { + console.log('DB Operation: Updating Test Shot table with OCR data'); + try { + return await db.tx(t => { + var queries = concatenatedOcrTextArr.map(ocrObj => { + return t.one('UPDATE test_shot SET original_ocr = ${text} WHERE shot_id = ${shot_id} RETURNING id', ocrObj, a => + a.id); + }); + return t.batch(queries); + }); + } catch(e) { + throw e; + } + } + + async insertIntoTest_ShotTable(shotArr) { + console.log('DB Operation: Inserting Shots in test_shot'); + try { + return await db.tx(t => { + var queries = shotArr.map(shot => { + return t.one('INSERT INTO test_shot(shot_id, scene_id, shot_num, start_time, end_time, original_ocr, original_description) VALUES(${shot_id}, ${scene_id}, ${shot_num}, ${start_time}, ${end_time}, ${original_ocr}, ${original_description}) RETURNING id', shot, a => + a.id); + }); + return t.batch(queries); + }); + } catch(e) { + throw e; + } + } + + + concatOcrWithinSameShot(ocrArr) { + let outputArr = []; + let customOcrObj = new Object(); + customOcrObj.shot_id = '-'; + customOcrObj.text = ''; + let lastShotNum = -1; + for(let i=0; i lastShotNum) { + if(lastShotNum > -1) { + outputArr.push(customOcrObj); + } + customOcrObj = new Object(); + customOcrObj.shot_id = currOcrObj.shot_id; + customOcrObj.text = ''; + lastShotNum = currOcrObj.shot_num; + } + customOcrObj.text = customOcrObj.text + currOcrObj.text + ' '; + } + customOcrObj.text = customOcrObj.text.trim(); + outputArr.push(customOcrObj); + return outputArr; + } + + //main logic for tweaking the duration of OCR + setShotIdForAllOcrDetections(ocrArr, shotArr) { + let shotStartTimeArr = this.extractStartTimeArrFromObjectArr(shotArr); + for (const ocrObj of ocrArr) { + let shotIndex = this.getIndexOfInputArrForTargetVal(ocrObj.start_time, shotStartTimeArr); + let parentShot = shotArr[shotIndex]; + ocrObj.shot_id = parentShot.shot_id; + ocrObj.shot_num = parentShot.shot_num; + ocrObj.shot_start_time = parentShot.start_time; + ocrObj.shot_end_time = parentShot.end_time; + ocrObj.diff = ocrObj.end_time - ocrObj.start_time; + } + + return ocrArr; + } + + + cleanUpOcrDataWithinEachShot(ocrArr) { + let ocrMap = new Map(); + for(let i=0; i { + outputArr.push(obj.start_time); + }); + return outputArr; + } + + //Returns index of array item whose value is equal-to or lesser-than(also nearest-in-value) the target (binary search algorithm) + getIndexOfInputArrForTargetVal(target, inputArr) { + let start = 0; + let end = inputArr.length - 1; + let index = -1; + while(start <= end) { + let mid = Math.floor((start + end)/2); + if(inputArr[mid] == target) { + index = mid; + break; + } else if(inputArr[mid] < target) { + index = mid; + start = mid + 1; + } else { + end = mid - 1; + } + } + return index; + } + +} + + + + +let controller = new OcrDuplicateRemovalController(); +// let file = './heinz.json'; +// let videoId = '64ad005466%'; + +// HOPE FOR PAWS +// file = './hopeForPaws.json'; +// videoId = 'll8cTIg2rwM%'; + +// BAWDY +// file = './bawdy.json'; +// videoId = '6ZLCmVlQBLo%'; + +//ocean8 +// file = './oceans8.json'; +// videoId = 'OkBaZLq7gnU%'; + +//Fried Potato Ball +file = './potatoBall.json'; +videoId= 'nqXz8hhAYGo%' + + +// CRISPY Cabbage +// file = './crispyCabbage.json'; +// videoId= 'cNj3aOTYdQQ%' + +//MISSION IMPOSSIBLE ROUGE +// file = './MissionImpossibleRouge.json'; +// videoId = 'elpUGB9Ap1Y%'; + + +let duration = 1; +let confidenceScore = 0.56 ; +let topLimit = 100; +console.log({videoId, duration, confidenceScore, topLimit}); +controller.removeDuplicate(file, videoId, duration, confidenceScore, topLimit); diff --git a/controllers/PythiaCaptionGenerationController.js b/controllers/PythiaCaptionGenerationController.js new file mode 100644 index 0000000..5e97213 --- /dev/null +++ b/controllers/PythiaCaptionGenerationController.js @@ -0,0 +1,88 @@ +const axios = require('axios'); +const sceneModel = require('../model/scene'); +const keyframeModel = require('../model/keyframe'); +const qs = require('qs') + + +class PythiaCaptionGenerationController{ + + static async generatePythiaCaption(videoId){ + try{ + let data = await keyframeModel.getKeyFramesByVideoId(videoId); + // console.log({data}); + let captionArr = []; + console.log('Fetching Pythia captions for keyframes.....'); + for(let i=0; i< data.length; i++){ + let thumbnail_url = data[i].thumbnail_url; + let keyframe_id = data[i].keyframe_id; + // console.log({thumbnail_url}); + let captionResponse = await this.fetchPythiaCaption(thumbnail_url); + // console.log({Pycaption: captionResponse.data}); + data[i].updatedPythiaCaption = captionResponse.data; + captionArr.push({keyframe_id, thumbnail_url, caption: captionResponse.data }); + } + // console.log({captionArr}); + // console.log({data}); + let response = await keyframeModel.updatePythiaCaption(captionArr); + // console.log(response); + + let pythiaCaptions = await keyframeModel.getKeyFramesByVideoId(videoId); + // console.log({caption: pythiaCaptions.updatedPythiaCaption}); + + let sceneMap = new Map(); + for(let i=0; i< data.length; i++){ + let sceneId = data[i].scene_id; + let caption = data[i].updatedPythiaCaption; + + let value = sceneMap.get(sceneId); + if(value){ + if(!value.includes(caption)){ + caption = value + '\n' + caption; + sceneMap.set(sceneId, caption); + } + }else{ + sceneMap.set(sceneId, caption); + } + } + + // console.log({sceneMap}); + let sceneCaptionArr = []; + for (let [key, value] of sceneMap) { + console.log(key + ' = ' + value) + sceneCaptionArr.push({sceneId:key, caption:value}); + } + // console.log({sceneCaptionArr}); + + await sceneModel.updatePythiaCaptionForAllScenes(sceneCaptionArr); + console.log('Successfully updated pythia caption in scene table'); + + }catch(e){ + console.log(e); + } + + } + + static async fetchPythiaCaption(image_url){ + try{ + let result = await axios({ + method: 'post', + url : 'http://ec2-34-221-19-208.us-west-2.compute.amazonaws.com:8080/api', + data: qs.stringify({ + token : 'ZSDL6MXc6urYauWzVLXnUgE8yZcfCncCYgNyX4jRRnrGCTLmV4KUKrNDYSbfWAf2', + image_url + }), + headers: {'Content-Type': 'application/x-www-form-urlencoded'} + }) + return result; + }catch(error){ + console.log(error); + throw error; + } + + } + +} + +module.exports = PythiaCaptionGenerationController; + +// PythiaCaptionGenerationController.generatePythiaCaption('GXWpEjEFWkM'); diff --git a/controllers/VideoControllerUpload.js b/controllers/VideoControllerUpload.js new file mode 100644 index 0000000..08e062c --- /dev/null +++ b/controllers/VideoControllerUpload.js @@ -0,0 +1,444 @@ +const fs = require('fs'); +const Utils = require('../backend/utils'); +const db = require('../db/database'); +var ffmpeg = require('ffmpeg'); +let videoModel = require('../model/video.js'); +const youtubedl = require('youtube-dl'); + +const axios = require('axios'); +var ffmpeg = require('ffmpeg'); +const scene = require('../model/scene'); +const videoDialogue = require('../model/videoDialogue'); +const uploadVideoRoute = require('../routes/uploadVideo'); +var path = require('path'); +let filterDescController = require('../controllers/FilterBaselineDescriptionController'); +let pythiaCaptionController = require('../controllers/PythiaCaptionGenerationController'); + + +class UploadVideoController{ + + + static async insertAudioInsightIntoVideo_Dialogue(){ + let videoId = 'vBkBS4O3yvY'; + let audioId = 'OMHPXC20200411053439'; + try{ + let url = 'http://18.221.192.73:8081/v1/audioclips/getspeechtotextresult?orderid=' + audioId; + let response = await axios.get(url); + console.log(response.data); + + let audioInsight =[]; + let sentence = response.data.result.sentence; + for(let i=0; i< sentence.length; i++){ + let start = sentence[i].startTime; + let end = sentence[i].endTime; + let desc = sentence[i].sentence; + desc = desc.replace(/'/g, ''); + if(!desc.toLowerCase().includes('noise')){ + let data = { + start, + end, + sentence: desc + } + audioInsight.push(data); + } + } + console.log({audioInsight}); + let result = await videoDialogue.insertDataIntoVideoDialogue(audioInsight, videoId, audioId); + console.log({result}); + + }catch(e){ + console.log(e); + } + + + } + + static async getAudioInsightAndMerge(audioId, videoId){ + // let data = { 'NDAUMF20200408105803': 'OkBaZLq7gnU'}; + // data = { audioId: 'NZTJTD20200408120207', youtubeId: '7QZNpS0uos4' } + // data = { audioId: 'POFZRY20200408120928', youtubeId: 'elpUGB9Ap1Y' } + //{ audioId: 'EFUPVK20200408121751', youtubeId: '2wwC9c3iYK4' } + + return this.fetchAudioInsight(audioId, videoId) + .then((result)=>{ + console.log(result); + }) + .catch(e => { + console.log(e); + }) + } + + + static async fetchAndFilterPythiaCaption(videoId){ + try{ + let captionRes = await pythiaCaptionController.generatePythiaCaption(videoId) + console.log('PythiaCaption:', captionRes); + await filterDescController.filter(videoId); // filter original description + let res = filterDescController.filterPythiaCaption(videoId); + return res; + }catch(e){ + console.log(e); + throw e; + } + } + + static async fetchAudioInsight(audioId, videoId){ + return new Promise((resolve, reject) => { + let url = 'http://18.221.192.73:8081/v1/audioclips/getspeechtotextresult?orderid=' + audioId; + axios.get(url) + .then((response) => { + console.log(response.data); + let status = response.data.status + if(status !== 'success'){ + reject(`Audio not processed for audioId=${audioId}, videoId= ${videoId}`); + } + //VIDEO + let videoInsight = []; + console.log({videoId}); + scene.getSceneData(videoId) + .then((data) => { + console.log({sceneData: data}); + for (let i = 0; i < data.length; i++) { + data[i].merged = []; + videoInsight.push(data[i]); + } + console.log({ videoInsight }); + + + //AUDIO + let sentence = response.data.result.sentence; + let audioInsight =[]; + for(let i=0; i< sentence.length; i++){ + let start = sentence[i].startTime; + let end = sentence[i].endTime; + let desc = sentence[i].sentence; + desc = desc.replace(/'/g, ''); + if(!desc.toLowerCase().includes('noise')){ + let data = { + start, + end, + sentence: desc + } + audioInsight.push(data); + } + } + console.log({audioInsight}); + + //MERGE + if(videoInsight.length == 0){ + console.log('Scene data empty for the video'); + reject(); + } + + let mergedList = this.merge(videoInsight, audioInsight); + scene.addMergeSceneTable(mergedList) + .then(data => { + console.log({ success: data }); + videoDialogue.insertDataIntoVideoDialogue(audioInsight, videoId, audioId) + .then(res=>{ + console.log(res); + resolve(); + }) + }).catch((error) => { + console.log(error); + reject(error); + }) + + }) + + }) + .catch((error) => { + console.log(error); + reject(error); + }) + }); + + } + + + + static merge(videoInsight, audioInsight){ + let videoIndex = 0; + let mergedList = []; + mergedList.push(videoInsight[videoIndex++]); + let i = 0; + while(i < audioInsight.length && videoIndex < videoInsight.length){ + let size = mergedList.length; + let s1 = mergedList[size-1].start_time; + let e1 = mergedList[size-1].end_time; + let description = mergedList[size-1].original_description; + let ocr = mergedList[size-1].original_ocr; + + let s2 = audioInsight[i].start; + let e2 = audioInsight[i].end; + + if((s2 >= s1 && s2 <= e1) && (e2 >= s1 && e2 <= e1)) { + i++; + continue; + } + + if(Math.min(s2, e2) >= e1) { + mergedList.push(videoInsight[videoIndex++]); + continue; + } + + //merge condition + if(s2 >= s1 && e2 > e1) { + let sceneObj = mergedList[mergedList.length-1]; + let next = videoInsight[videoIndex++]; + let min = Math.min(s1, next.start_time); + let max = Math.max(e1, next.end_time); + sceneObj.start_time = min; + sceneObj.end_time = max; + sceneObj.original_description = description + next.original_description; + sceneObj.original_ocr = ocr + next.original_ocr; + + // let arr = sceneObj.merged; + // arr.push(next.scene_id); + sceneObj.merged.push(next.scene_id); + mergedList.splice(mergedList.length-1); + mergedList.push(sceneObj); + }else if(s2 >= e1) { + mergedList.push(videoInsight[videoIndex++]); + } + + i++; + } + + for(let j = videoIndex; j', end); + } + console.log('video:', arr); + + return arr; + } + + static parseAudioFile(file){ + console.log("here"); + let rawdata = fs.readFileSync(file); + let obj = JSON.parse(rawdata); + let sentence = obj.result.sentence; + let arr =[]; + for(let i=0; i< sentence.length; i++){ + // let instance = scenes[i].instances[0]; + let start = sentence[i].startTime; + let end = sentence[i].endTime; + // let end = Utils.parseSecondsFromString(instance.end); + // console.log(start,'->', end); + // console.log(start , ',' , end); + let data = { + start, + end + } + arr.push(data); + } + + console.log(arr); + return arr; + } + + + static convertVideoToAudio(path, fileName){ + let audioFile = path + '/' + fileName+'.mp3'; + let videoFile = path + '/' + fileName + '.mp4' + console.log({path}); + console.log({audioFile}); + console.log({videoFile}); + + try { + var process = new ffmpeg(videoFile); + process.then(function (video) { + console.log({video}); + video.fnExtractSoundToMP3(audioFile, function (error, file) { + if (!error) + console.log('Audio file: ' + file); + }); + }, function (err) { + console.log('Error: ' + err); + }); + } catch (e) { + console.log('err->', e); + } + } + + + static async insertIntoVideoDialogue1(){ + let audioId = 'OMHPXC20200411053439'; + let videoId = 'vBkBS4O3yvY'; + + try{ + + let url = 'http://18.221.192.73:8081/v1/audioclips/getspeechtotextresult?orderid=' + audioId; + let response = await axios.get(url); + + //AUDIO + let sentence = response.data.result.sentence; + let audioInsight =[]; + for(let i=0; i< sentence.length; i++){ + let start = sentence[i].startTime; + let end = sentence[i].endTime; + let desc = sentence[i].sentence; + desc = desc.replace(/'/g, ''); + + let data = { + start, + end, + sentence: desc + } + audioInsight.push(data); + } + console.log({audioInsight}); + + await videoDialogue.insertDataIntoVideoDialogue(audioInsight, videoId, audioId) + + console.log('inserted'); + }catch (e){ + console.log(e); + } + + + + + } + + + async copyFromMergeToDescription(user, videoId){ + // let videoId = 'vBkBS4O3yvY'; + // let user = 'd32f61aa-7f67-11ea-bc55-0242ac130003'; + + console.log({videoId}); + try { + let data = await db.query('select * from merge_scene where video_id = $1 order by start_time', [videoId]); + // console.log(data); + console.log('here'); + let sql = ''; + for(let i=0; iELUNG","confidence":0.273,"left":178,"top":100,"width":356,"height":49,"language":"en-US","instances":[{"adjustedStart":"0:01:51.178","adjustedEnd":"0:01:51.645","start":"0:01:51.178","end":"0:01:51.645"}]},{"id":30,"text":"STOR$ELUNG","confidence":0.2652,"left":178,"top":100,"width":356,"height":53,"language":"en-US","instances":[{"adjustedStart":"0:04:22.062","adjustedEnd":"0:04:22.996","start":"0:04:22.062","end":"0:04:22.996"},{"adjustedStart":"0:11:37.43","adjustedEnd":"0:11:37.897","start":"0:11:37.43","end":"0:11:37.897"}]},{"id":31,"text":"STOÜELUNG","confidence":0.263,"left":178,"top":100,"width":356,"height":49,"language":"en-US","instances":[{"adjustedStart":"0:05:01.301","adjustedEnd":"0:05:01.768","start":"0:05:01.301","end":"0:05:01.768"}]},{"id":32,"text":"STOR±TELUNG","confidence":0.2722,"left":178,"top":100,"width":356,"height":57,"language":"en-US","instances":[{"adjustedStart":"0:06:06.233","adjustedEnd":"0:06:06.7","start":"0:06:06.233","end":"0:06:06.7"},{"adjustedStart":"0:06:07.167","adjustedEnd":"0:06:08.568","start":"0:06:07.167","end":"0:06:08.568"}]},{"id":33,"text":"STOÜYELUNG","confidence":0.2658,"left":178,"top":100,"width":356,"height":49,"language":"en-US","instances":[{"adjustedStart":"0:06:25.385","adjustedEnd":"0:06:27.253","start":"0:06:25.385","end":"0:06:27.253"},{"adjustedStart":"0:06:28.188","adjustedEnd":"0:06:29.122","start":"0:06:28.188","end":"0:06:29.122"}]},{"id":34,"text":"STOR'?iELUNG","confidence":0.2613,"left":178,"top":100,"width":356,"height":57,"language":"en-US","instances":[{"adjustedStart":"0:06:45.005","adjustedEnd":"0:06:45.939","start":"0:06:45.005","end":"0:06:45.939"},{"adjustedStart":"0:06:46.406","adjustedEnd":"0:06:47.34","start":"0:06:46.406","end":"0:06:47.34"},{"adjustedStart":"0:06:48.275","adjustedEnd":"0:06:48.742","start":"0:06:48.275","end":"0:06:48.742"}]},{"id":35,"text":"STORY3ELUNG","confidence":0.2728,"left":178,"top":100,"width":356,"height":49,"language":"en-US","instances":[{"adjustedStart":"0:07:00.42","adjustedEnd":"0:07:00.887","start":"0:07:00.42","end":"0:07:00.887"},{"adjustedStart":"0:07:02.289","adjustedEnd":"0:07:02.756","start":"0:07:02.289","end":"0:07:02.756"},{"adjustedStart":"0:07:04.157","adjustedEnd":"0:07:04.624","start":"0:07:04.157","end":"0:07:04.624"}]},{"id":36,"text":"STOÉT?ELUNG","confidence":0.2665,"left":178,"top":100,"width":356,"height":53,"language":"en-US","instances":[{"adjustedStart":"0:07:20.04","adjustedEnd":"0:07:20.974","start":"0:07:20.04","end":"0:07:20.974"},{"adjustedStart":"0:07:23.31","adjustedEnd":"0:07:23.777","start":"0:07:23.31","end":"0:07:23.777"}]},{"id":37,"text":"STORY*ELUNG","confidence":0.2675,"left":178,"top":100,"width":356,"height":52,"language":"en-US","instances":[{"adjustedStart":"0:07:32.185","adjustedEnd":"0:07:32.652","start":"0:07:32.185","end":"0:07:32.652"},{"adjustedStart":"0:07:34.054","adjustedEnd":"0:07:34.521","start":"0:07:34.054","end":"0:07:34.521"}]},{"id":38,"text":"STORtTELUNG","confidence":0.251,"left":178,"top":100,"width":356,"height":46,"language":"en-US","instances":[{"adjustedStart":"0:09:20.56","adjustedEnd":"0:09:21.027","start":"0:09:20.56","end":"0:09:21.027"}]},{"id":39,"text":"STOR#tELUNG","confidence":0.2736,"left":178,"top":100,"width":356,"height":49,"language":"en-US","instances":[{"adjustedStart":"0:09:25.231","adjustedEnd":"0:09:25.698","start":"0:09:25.231","end":"0:09:25.698"},{"adjustedStart":"0:09:27.1","adjustedEnd":"0:09:27.567","start":"0:09:27.1","end":"0:09:27.567"}]},{"id":40,"text":"STORYTELUNG","confidence":0.2828,"left":178,"top":100,"width":356,"height":49,"language":"en-US","instances":[{"adjustedStart":"0:09:36.443","adjustedEnd":"0:09:36.91","start":"0:09:36.443","end":"0:09:36.91"},{"adjustedStart":"0:09:38.778","adjustedEnd":"0:09:39.712","start":"0:09:38.778","end":"0:09:39.712"}]},{"id":41,"text":"STOR$ÄELUNG","confidence":0.2596,"left":178,"top":100,"width":356,"height":57,"language":"en-US","instances":[{"adjustedStart":"0:09:47.654","adjustedEnd":"0:09:49.055","start":"0:09:47.654","end":"0:09:49.055"},{"adjustedStart":"0:09:49.989","adjustedEnd":"0:09:50.456","start":"0:09:49.989","end":"0:09:50.456"}]},{"id":42,"text":"STORtYELUNG","confidence":0.2577,"left":178,"top":100,"width":356,"height":49,"language":"en-US","instances":[{"adjustedStart":"0:09:58.398","adjustedEnd":"0:09:59.332","start":"0:09:58.398","end":"0:09:59.332"}]},{"id":43,"text":"STOÉfTELUNG","confidence":0.2706,"left":178,"top":100,"width":356,"height":56,"language":"en-US","instances":[{"adjustedStart":"0:10:05.405","adjustedEnd":"0:10:05.872","start":"0:10:05.405","end":"0:10:05.872"},{"adjustedStart":"0:10:06.339","adjustedEnd":"0:10:07.74","start":"0:10:06.339","end":"0:10:07.74"},{"adjustedStart":"0:10:09.142","adjustedEnd":"0:10:10.076","start":"0:10:09.142","end":"0:10:10.076"}]},{"id":44,"text":"STORf$ELUNG","confidence":0.275,"left":178,"top":100,"width":356,"height":50,"language":"en-US","instances":[{"adjustedStart":"0:10:18.017","adjustedEnd":"0:10:18.484","start":"0:10:18.017","end":"0:10:18.484"}]},{"id":45,"text":"STOKöTELUNG","confidence":0.258,"left":178,"top":100,"width":356,"height":50,"language":"en-US","instances":[{"adjustedStart":"0:11:30.89","adjustedEnd":"0:11:31.357","start":"0:11:30.89","end":"0:11:31.357"},{"adjustedStart":"0:11:31.824","adjustedEnd":"0:11:32.291","start":"0:11:31.824","end":"0:11:32.291"},{"adjustedStart":"0:11:32.759","adjustedEnd":"0:11:33.226","start":"0:11:32.759","end":"0:11:33.226"}]},{"id":46,"text":"STOKÄELUNG","confidence":0.252,"left":178,"top":100,"width":356,"height":48,"language":"en-US","instances":[{"adjustedStart":"0:11:46.306","adjustedEnd":"0:11:46.773","start":"0:11:46.306","end":"0:11:46.773"},{"adjustedStart":"0:11:47.24","adjustedEnd":"0:11:47.707","start":"0:11:47.24","end":"0:11:47.707"}]},{"id":47,"text":"STORIÄELUNG","confidence":0.253,"left":178,"top":100,"width":356,"height":52,"language":"en-US","instances":[{"adjustedStart":"0:11:53.78","adjustedEnd":"0:11:54.247","start":"0:11:53.78","end":"0:11:54.247"}]},{"id":48,"text":"STOR%'ELUNG","confidence":0.264,"left":178,"top":100,"width":356,"height":56,"language":"en-US","instances":[{"adjustedStart":"0:12:19.939","adjustedEnd":"0:12:20.406","start":"0:12:19.939","end":"0:12:20.406"}]},{"id":49,"text":"STORV:ELUNG","confidence":0.274,"left":178,"top":100,"width":356,"height":48,"language":"en-US","instances":[{"adjustedStart":"0:12:20.406","adjustedEnd":"0:12:20.873","start":"0:12:20.406","end":"0:12:20.873"}]},{"id":50,"text":"STOMTiELUNG","confidence":0.2836,"left":178,"top":100,"width":356,"height":52,"language":"en-US","instances":[{"adjustedStart":"0:12:25.545","adjustedEnd":"0:12:26.012","start":"0:12:25.545","end":"0:12:26.012"},{"adjustedStart":"0:12:26.946","adjustedEnd":"0:12:27.413","start":"0:12:26.946","end":"0:12:27.413"},{"adjustedStart":"0:12:28.815","adjustedEnd":"0:12:29.282","start":"0:12:28.815","end":"0:12:29.282"}]},{"id":51,"text":"STOR#'TELUNG","confidence":0.265,"left":178,"top":100,"width":356,"height":48,"language":"en-US","instances":[{"adjustedStart":"0:12:35.822","adjustedEnd":"0:12:36.756","start":"0:12:35.822","end":"0:12:36.756"},{"adjustedStart":"0:12:37.223","adjustedEnd":"0:12:37.69","start":"0:12:37.223","end":"0:12:37.69"},{"adjustedStart":"0:12:38.157","adjustedEnd":"0:12:39.092","start":"0:12:38.157","end":"0:12:39.092"}]},{"id":52,"text":"STORt*ELUNG","confidence":0.258,"left":178,"top":100,"width":356,"height":51,"language":"en-US","instances":[{"adjustedStart":"0:12:53.106","adjustedEnd":"0:12:54.04","start":"0:12:53.106","end":"0:12:54.04"}]},{"id":53,"text":"STOKfrELUNG","confidence":0.2655,"left":178,"top":100,"width":356,"height":50,"language":"en-US","instances":[{"adjustedStart":"0:12:58.244","adjustedEnd":"0:12:59.178","start":"0:12:58.244","end":"0:12:59.178"},{"adjustedStart":"0:12:59.646","adjustedEnd":"0:13:00.113","start":"0:12:59.646","end":"0:13:00.113"}]},{"id":54,"text":"STOm$ELUNG","confidence":0.268,"left":182,"top":100,"width":355,"height":61,"language":"en-US","instances":[{"adjustedStart":"0:07:08.828","adjustedEnd":"0:07:09.295","start":"0:07:08.828","end":"0:07:09.295"}]},{"id":55,"text":"STORI?ELUNG","confidence":0.2562,"left":182,"top":100,"width":355,"height":58,"language":"en-US","instances":[{"adjustedStart":"0:11:16.876","adjustedEnd":"0:11:17.81","start":"0:11:16.876","end":"0:11:17.81"},{"adjustedStart":"0:11:19.679","adjustedEnd":"0:11:20.146","start":"0:11:19.679","end":"0:11:20.146"}]},{"id":56,"text":"TORTE LUNG","confidence":0.285,"left":209,"top":100,"width":326,"height":50,"language":"en-US","instances":[{"adjustedStart":"0:11:22.482","adjustedEnd":"0:11:22.949","start":"0:11:22.482","end":"0:11:22.949"}]},{"id":57,"text":"STOR#FELUNG","confidence":0.2621,"left":178,"top":101,"width":356,"height":50,"language":"en-US","instances":[{"adjustedStart":"0:01:56.783","adjustedEnd":"0:01:57.25","start":"0:01:56.783","end":"0:01:57.25"},{"adjustedStart":"0:01:57.718","adjustedEnd":"0:01:58.185","start":"0:01:57.718","end":"0:01:58.185"},{"adjustedStart":"0:01:58.652","adjustedEnd":"0:01:59.119","start":"0:01:58.652","end":"0:01:59.119"}]},{"id":58,"text":"STOC\\TELUNG","confidence":0.252,"left":178,"top":101,"width":356,"height":49,"language":"en-US","instances":[{"adjustedStart":"0:06:18.845","adjustedEnd":"0:06:19.312","start":"0:06:18.845","end":"0:06:19.312"}]},{"id":59,"text":"STOR3rELUNG","confidence":0.2823,"left":178,"top":101,"width":356,"height":45,"language":"en-US","instances":[{"adjustedStart":"0:08:55.802","adjustedEnd":"0:08:56.736","start":"0:08:55.802","end":"0:08:56.736"},{"adjustedStart":"0:08:57.67","adjustedEnd":"0:08:58.137","start":"0:08:57.67","end":"0:08:58.137"},{"adjustedStart":"0:08:59.072","adjustedEnd":"0:08:59.539","start":"0:08:59.072","end":"0:08:59.539"}]},{"id":60,"text":"STOOTELUNG","confidence":0.2687,"left":178,"top":101,"width":356,"height":49,"language":"en-US","instances":[{"adjustedStart":"0:09:18.224","adjustedEnd":"0:09:19.158","start":"0:09:18.224","end":"0:09:19.158"}]},{"id":61,"text":"STORt&ELUNG","confidence":0.2714,"left":177,"top":102,"width":356,"height":54,"language":"en-US","instances":[{"adjustedStart":"0:06:36.129","adjustedEnd":"0:06:38.465","start":"0:06:36.129","end":"0:06:38.465"},{"adjustedStart":"0:06:38.932","adjustedEnd":"0:06:40.333","start":"0:06:38.932","end":"0:06:40.333"}]},{"id":62,"text":"STOR3ELUNG","confidence":0.257,"left":178,"top":102,"width":356,"height":49,"language":"en-US","instances":[{"adjustedStart":"0:02:36.49","adjustedEnd":"0:02:36.957","start":"0:02:36.49","end":"0:02:36.957"},{"adjustedStart":"0:02:38.825","adjustedEnd":"0:02:39.292","start":"0:02:38.825","end":"0:02:39.292"}]},{"id":63,"text":"STORY* LUNG","confidence":0.3562,"left":178,"top":102,"width":356,"height":56,"language":"en-US","instances":[{"adjustedStart":"0:07:25.645","adjustedEnd":"0:07:26.112","start":"0:07:25.645","end":"0:07:26.112"},{"adjustedStart":"0:09:33.173","adjustedEnd":"0:09:33.64","start":"0:09:33.173","end":"0:09:33.64"},{"adjustedStart":"0:12:26.479","adjustedEnd":"0:12:26.946","start":"0:12:26.479","end":"0:12:26.946"}]},{"id":64,"text":"STOÉT:rELUNG","confidence":0.269,"left":178,"top":102,"width":356,"height":58,"language":"en-US","instances":[{"adjustedStart":"0:09:40.647","adjustedEnd":"0:09:41.114","start":"0:09:40.647","end":"0:09:41.114"},{"adjustedStart":"0:09:41.581","adjustedEnd":"0:09:42.982","start":"0:09:41.581","end":"0:09:42.982"}]},{"id":65,"text":"STOR'?TELUNG","confidence":0.252,"left":178,"top":104,"width":356,"height":50,"language":"en-US","instances":[{"adjustedStart":"0:05:17.651","adjustedEnd":"0:05:18.118","start":"0:05:17.651","end":"0:05:18.118"}]},{"id":66,"text":"STOR","confidence":0.521,"left":152,"top":270,"width":338,"height":32,"language":"en-US","instances":[{"adjustedStart":"0:02:09.863","adjustedEnd":"0:02:10.33","start":"0:02:09.863","end":"0:02:10.33"}]},{"id":62,"text":"o is ä trooper and•he is","confidence":0.8083,"left":213,"top":270,"width":304,"height":27,"language":"en-US","instances":[{"adjustedStart":"0:02:34.154","adjustedEnd":"0:02:34.621","start":"0:02:34.154","end":"0:02:34.621"}]},{"id":63,"text":"o.is a trooperand he .is","confidence":0.7376,"left":213,"top":270,"width":304,"height":27,"language":"en-US","instances":[{"adjustedStart":"0:02:35.555","adjustedEnd":"0:02:36.022","start":"0:02:35.555","end":"0:02:36.022"},{"adjustedStart":"0:02:36.49","adjustedEnd":"0:02:36.957","start":"0:02:36.49","end":"0:02:36.957"}]},{"id":64,"text":"Q is a trooper and he is","confidence":0.8747,"left":292,"top":270,"width":225,"height":27,"language":"en-US","instances":[{"adjustedStart":"0:02:32.285","adjustedEnd":"0:02:32.752","start":"0:02:32.285","end":"0:02:32.752"},{"adjustedStart":"0:02:33.687","adjustedEnd":"0:02:34.154","start":"0:02:33.687","end":"0:02:34.154"},{"adjustedStart":"0:02:34.621","adjustedEnd":"0:02:35.088","start":"0:02:34.621","end":"0:02:35.088"}]},{"id":65,"text":"Milo","confidence":0.477,"left":443,"top":270,"width":54,"height":25,"language":"en-US","instances":[{"adjustedStart":"0:02:44.898","adjustedEnd":"0:02:45.365","start":"0:02:44.898","end":"0:02:45.365"}]},{"id":66,"text":"Even with a bandage,","confidence":0.8015,"left":146,"top":272,"width":260,"height":28,"language":"en-US","instances":[{"adjustedStart":"0:02:42.095","adjustedEnd":"0:02:46.766","start":"0:02:42.095","end":"0:02:46.766"}]},{"id":67,"text":"was","confidence":0.7518,"left":458,"top":275,"width":32,"height":17,"language":"en-US","instances":[{"adjustedStart":"0:02:08.929","adjustedEnd":"0:02:09.863","start":"0:02:08.929","end":"0:02:09.863"}]},{"id":68,"text":"Subscribe","confidence":0.6841,"left":80,"top":278,"width":116,"height":20,"language":"en-US","instances":[{"adjustedStart":"0:00:03.27","adjustedEnd":"0:00:05.605","start":"0:00:03.27","end":"0:00:05.605"}]},{"id":69,"text":"Stibscribe","confidence":0.5583,"left":80,"top":278,"width":116,"height":20,"language":"en-US","instances":[{"adjustedStart":"0:03:30.21","adjustedEnd":"0:03:31.611","start":"0:03:30.21","end":"0:03:31.611"}]},{"id":70,"text":"but clearly in diktre•s","confidence":0.7812,"left":177,"top":287,"width":278,"height":41,"language":"en-US","instances":[{"adjustedStart":"0:00:33.634","adjustedEnd":"0:00:34.101","start":"0:00:33.634","end":"0:00:34.101"}]},{"id":71,"text":"SB","confidence":0.576,"left":314,"top":289,"width":23,"height":15,"language":"en-US","instances":[{"adjustedStart":"0:00:03.737","adjustedEnd":"0:00:04.204","start":"0:00:03.737","end":"0:00:04.204"},{"adjustedStart":"0:00:04.671","adjustedEnd":"0:00:05.138","start":"0:00:04.671","end":"0:00:05.138"}]},{"id":72,"text":"ampunt8puS from","confidence":0.635,"left":84,"top":292,"width":256,"height":40,"language":"en-US","instances":[{"adjustedStart":"0:02:03.323","adjustedEnd":"0:02:03.79","start":"0:02:03.323","end":"0:02:03.79"}]},{"id":73,"text":"amoun *pus from","confidence":0.6407,"left":84,"top":293,"width":256,"height":35,"language":"en-US","instances":[{"adjustedStart":"0:02:02.856","adjustedEnd":"0:02:03.323","start":"0:02:02.856","end":"0:02:03.323"}]},{"id":74,"text":"Suffering froYlfå bite wound.","confidence":0.7242,"left":136,"top":294,"width":368,"height":34,"language":"en-US","instances":[{"adjustedStart":"0:02:11.732","adjustedEnd":"0:02:12.199","start":"0:02:11.732","end":"0:02:12.199"}]},{"id":75,"text":"butueåly in distie s.","confidence":0.644,"left":177,"top":296,"width":285,"height":32,"language":"en-US","instances":[{"adjustedStart":"0:00:32.232","adjustedEnd":"0:00:32.699","start":"0:00:32.232","end":"0:00:32.699"}]},{"id":76,"text":"amount•frpüs ftom","confidence":0.5685,"left":84,"top":297,"width":257,"height":31,"language":"en-US","instances":[{"adjustedStart":"0:02:02.389","adjustedEnd":"0:02:02.856","start":"0:02:02.389","end":"0:02:02.856"}]},{"id":77,"text":"He knew we were there toelp im","confidence":0.8996,"left":83,"top":298,"width":329,"height":25,"language":"en-US","instances":[{"adjustedStart":"0:01:08.669","adjustedEnd":"0:01:09.603","start":"0:01:08.669","end":"0:01:09.603"}]},{"id":78,"text":"amount*öfrpu from","confidence":0.7085,"left":84,"top":298,"width":256,"height":33,"language":"en-US","instances":[{"adjustedStart":"0:02:01.922","adjustedEnd":"0:02:02.389","start":"0:02:01.922","end":"0:02:02.389"}]},{"id":79,"text":"•rescues like this are possible.","confidence":0.8814,"left":85,"top":298,"width":430,"height":29,"language":"en-US","instances":[{"adjustedStart":"0:03:15.262","adjustedEnd":"0:03:15.729","start":"0:03:15.262","end":"0:03:15.729"}]},{"id":80,"text":"rescues like this are possible.","confidence":0.8646,"left":126,"top":298,"width":389,"height":23,"language":"en-US","instances":[{"adjustedStart":"0:03:12.459","adjustedEnd":"0:03:14.794","start":"0:03:12.459","end":"0:03:14.794"}]},{"id":81,"text":"www.CauseForSBPaws.org","confidence":0.6348,"left":135,"top":299,"width":368,"height":29,"language":"en-US","instances":[{"adjustedStart":"0:02:58.912","adjustedEnd":"0:03:03.116","start":"0:02:58.912","end":"0:03:03.116"}]},{"id":82,"text":"Serff€ring m bite wound","confidence":0.7185,"left":136,"top":299,"width":359,"height":35,"language":"en-US","instances":[{"adjustedStart":"0:02:09.863","adjustedEnd":"0:02:10.33","start":"0:02:09.863","end":"0:02:10.33"}]},{"id":83,"text":"but clearlysiriWdistreys.","confidence":0.7005,"left":177,"top":299,"width":285,"height":34,"language":"en-US","instances":[{"adjustedStart":"0:00:31.765","adjustedEnd":"0:00:32.232","start":"0:00:31.765","end":"0:00:32.232"}]},{"id":84,"text":"and a bath,","confidence":0.9547,"left":69,"top":300,"width":147,"height":26,"language":"en-US","instances":[{"adjustedStart":"0:02:22.009","adjustedEnd":"0:02:26.213","start":"0:02:22.009","end":"0:02:26.213"}]},{"id":85,"text":"to the call and ru hed to the location.","confidence":0.8675,"left":74,"top":300,"width":490,"height":24,"language":"en-US","instances":[{"adjustedStart":"0:00:10.277","adjustedEnd":"0:00:14.014","start":"0:00:10.277","end":"0:00:14.014"}]},{"id":86,"text":"He knew we were there to help im.","confidence":0.8084,"left":83,"top":300,"width":475,"height":28,"language":"en-US","instances":[{"adjustedStart":"0:01:10.07","adjustedEnd":"0:01:11.004","start":"0:01:10.07","end":"0:01:11.004"},{"adjustedStart":"0:01:11.471","adjustedEnd":"0:01:11.938","start":"0:01:11.471","end":"0:01:11.938"}]},{"id":87,"text":"to their adopted senio r dog,","confidence":0.8616,"left":87,"top":300,"width":365,"height":28,"language":"en-US","instances":[{"adjustedStart":"0:03:05.452","adjustedEnd":"0:03:08.722","start":"0:03:05.452","end":"0:03:08.722"}]},{"id":88,"text":"to their adopted senior dog, Gracé.","confidence":0.9009,"left":88,"top":300,"width":364,"height":28,"language":"en-US","instances":[{"adjustedStart":"0:03:03.583","adjustedEnd":"0:03:05.452","start":"0:03:03.583","end":"0:03:05.452"}]},{"id":89,"text":"guirney and","confidence":0.7901,"left":97,"top":300,"width":147,"height":28,"language":"en-US","instances":[{"adjustedStart":"0:01:05.399","adjustedEnd":"0:01:05.866","start":"0:01:05.399","end":"0:01:05.866"},{"adjustedStart":"0:01:07.267","adjustedEnd":"0:01:08.201","start":"0:01:07.267","end":"0:01:08.201"}]},{"id":90,"text":"now happy to receive affection.","confidence":0.8759,"left":115,"top":300,"width":410,"height":29,"language":"en-US","instances":[{"adjustedStart":"0:02:32.285","adjustedEnd":"0:02:33.22","start":"0:02:32.285","end":"0:02:33.22"},{"adjustedStart":"0:02:33.687","adjustedEnd":"0:02:37.424","start":"0:02:33.687","end":"0:02:37.424"}]},{"id":91,"text":"me that he was in a lot of pain.","confidence":0.8775,"left":118,"top":300,"width":405,"height":28,"language":"en-US","instances":[{"adjustedStart":"0:01:00.727","adjustedEnd":"0:01:01.661","start":"0:01:00.727","end":"0:01:01.661"},{"adjustedStart":"0:01:02.129","adjustedEnd":"0:01:02.596","start":"0:01:02.129","end":"0:01:02.596"}]},{"id":92,"text":"but he had a serious infection.","confidence":0.9663,"left":121,"top":300,"width":398,"height":23,"language":"en-US","instances":[{"adjustedStart":"0:01:53.981","adjustedEnd":"0:01:58.185","start":"0:01:53.981","end":"0:01:58.185"}]},{"id":93,"text":"to figure out what was wrong.","confidence":0.8025,"left":125,"top":300,"width":389,"height":29,"language":"en-US","instances":[{"adjustedStart":"0:01:30.624","adjustedEnd":"0:01:34.361","start":"0:01:30.624","end":"0:01:34.361"}]},{"id":94,"text":"su ering rom - bitew nd.","confidence":0.6227,"left":136,"top":300,"width":367,"height":29,"language":"en-US","instances":[{"adjustedStart":"0:02:09.396","adjustedEnd":"0:02:09.863","start":"0:02:09.396","end":"0:02:09.863"}]},{"id":95,"text":"suffer.in-gfrörn•au •tecwound.","confidence":0.3425,"left":136,"top":300,"width":367,"height":29,"language":"en-US","instances":[{"adjustedStart":"0:02:10.33","adjustedEnd":"0:02:10.797","start":"0:02:10.33","end":"0:02:10.797"}]},{"id":96,"text":"a-yard-and-then-collapsed.","confidence":0.626,"left":144,"top":300,"width":351,"height":28,"language":"en-US","instances":[{"adjustedStart":"0:00:07.007","adjustedEnd":"0:00:07.941","start":"0:00:07.007","end":"0:00:07.941"}]},{"id":97,"text":"a yard-anÜthGTCöllapsed.","confidence":0.5805,"left":144,"top":300,"width":351,"height":28,"language":"en-US","instances":[{"adjustedStart":"0:00:07.941","adjustedEnd":"0:00:08.408","start":"0:00:07.941","end":"0:00:08.408"}]},{"id":98,"text":"a yard-anu•thVffCölläPSéd.","confidence":0.553,"left":144,"top":300,"width":351,"height":28,"language":"en-US","instances":[{"adjustedStart":"0:00:08.408","adjustedEnd":"0:00:08.875","start":"0:00:08.408","end":"0:00:08.875"}]},{"id":99,"text":"www.HopseForSBPavÄ.org","confidence":0.5673,"left":164,"top":300,"width":312,"height":28,"language":"en-US","instances":[{"adjustedStart":"0:03:16.196","adjustedEnd":"0:03:17.13","start":"0:03:16.196","end":"0:03:17.13"},{"adjustedStart":"0:03:17.597","adjustedEnd":"0:03:19.466","start":"0:03:17.597","end":"0:03:19.466"}]},{"id":100,"text":"but clearly sin distress.","confidence":0.8471,"left":177,"top":300,"width":286,"height":29,"language":"en-US","instances":[{"adjustedStart":"0:00:29.429","adjustedEnd":"0:00:30.364","start":"0:00:29.429","end":"0:00:30.364"}]},{"id":101,"text":"but, -learly -in distréss.","confidence":0.7387,"left":177,"top":300,"width":285,"height":28,"language":"en-US","instances":[{"adjustedStart":"0:00:30.364","adjustedEnd":"0:00:31.765","start":"0:00:30.364","end":"0:00:31.765"},{"adjustedStart":"0:00:32.699","adjustedEnd":"0:00:33.633","start":"0:00:32.699","end":"0:00:33.633"}]},{"id":102,"text":"for","confidence":0.697,"left":258,"top":300,"width":18,"height":22,"language":"en-US","instances":[{"adjustedStart":"0:02:08.929","adjustedEnd":"0:02:09.396","start":"0:02:08.929","end":"0:02:09.396"}]},{"id":103,"text":"started feeling better.","confidence":0.8975,"left":292,"top":300,"width":278,"height":28,"language":"en-US","instances":[{"adjustedStart":"0:02:22.009","adjustedEnd":"0:02:26.213","start":"0:02:22.009","end":"0:02:26.213"}]},{"id":104,"text":"as cooperati","confidence":0.6825,"left":342,"top":300,"width":160,"height":28,"language":"en-US","instances":[{"adjustedStart":"0:01:07.267","adjustedEnd":"0:01:07.734","start":"0:01:07.267","end":"0:01:07.734"}]},{"id":105,"text":"o's sh IdeF.","confidence":0.4547,"left":407,"top":300,"width":148,"height":23,"language":"en-US","instances":[{"adjustedStart":"0:02:01.922","adjustedEnd":"0:02:02.389","start":"0:02:01.922","end":"0:02:02.389"},{"adjustedStart":"0:02:03.323","adjustedEnd":"0:02:03.79","start":"0:02:03.323","end":"0:02:03.79"}]},{"id":106,"text":"'s shoulder.","confidence":0.793,"left":407,"top":300,"width":148,"height":27,"language":"en-US","instances":[{"adjustedStart":"0:02:04.257","adjustedEnd":"0:02:04.724","start":"0:02:04.257","end":"0:02:04.724"}]},{"id":107,"text":"s ulder.","confidence":0.7745,"left":436,"top":300,"width":119,"height":23,"language":"en-US","instances":[{"adjustedStart":"0:02:02.856","adjustedEnd":"0:02:03.323","start":"0:02:02.856","end":"0:02:03.323"}]},{"id":108,"text":"gurney and","confidence":0.8704,"left":97,"top":301,"width":147,"height":28,"language":"en-US","instances":[{"adjustedStart":"0:01:03.997","adjustedEnd":"0:01:05.399","start":"0:01:03.997","end":"0:01:05.399"}]},{"id":109,"text":"was cooperative.","confidence":0.826,"left":319,"top":301,"width":223,"height":27,"language":"en-US","instances":[{"adjustedStart":"0:01:03.53","adjustedEnd":"0:01:05.399","start":"0:01:03.53","end":"0:01:05.399"}]},{"id":110,"text":"was co perative.","confidence":0.761,"left":319,"top":301,"width":223,"height":27,"language":"en-US","instances":[{"adjustedStart":"0:01:05.399","adjustedEnd":"0:01:05.866","start":"0:01:05.399","end":"0:01:05.866"}]},{"id":111,"text":"transformed into a happy, playful dog.","confidence":0.8811,"left":69,"top":302,"width":499,"height":29,"language":"en-US","instances":[{"adjustedStart":"0:02:42.095","adjustedEnd":"0:02:44.431","start":"0:02:42.095","end":"0:02:44.431"},{"adjustedStart":"0:02:44.898","adjustedEnd":"0:02:46.766","start":"0:02:44.898","end":"0:02:46.766"}]},{"id":112,"text":"wa","confidence":0.279,"left":319,"top":306,"width":37,"height":17,"language":"en-US","instances":[{"adjustedStart":"0:01:07.734","adjustedEnd":"0:01:08.201","start":"0:01:07.734","end":"0:01:08.201"}]}],"keywords":[{"id":1,"text":"nature brought","confidence":0.9975,"language":"en-US","instances":[{"adjustedStart":"0:03:03.583","adjustedEnd":"0:03:05.452","start":"0:03:03.583","end":"0:03:05.452"},{"adjustedStart":"0:03:05.452","adjustedEnd":"0:03:08.722","start":"0:03:05.452","end":"0:03:08.722"}]},{"id":2,"text":"bite wound","confidence":0.9975,"language":"en-US","instances":[{"adjustedStart":"0:02:11.732","adjustedEnd":"0:02:12.199","start":"0:02:11.732","end":"0:02:12.199"},{"adjustedStart":"0:02:09.863","adjustedEnd":"0:02:10.33","start":"0:02:09.863","end":"0:02:10.33"}]},{"id":3,"text":"adopted senio","confidence":0.9975,"language":"en-US","instances":[{"adjustedStart":"0:03:05.452","adjustedEnd":"0:03:08.722","start":"0:03:05.452","end":"0:03:08.722"},{"adjustedStart":"0:03:03.583","adjustedEnd":"0:03:05.452","start":"0:03:03.583","end":"0:03:05.452"}]},{"id":4,"text":"pain meds","confidence":0.9954,"language":"en-US","instances":[{"adjustedStart":"0:02:22.009","adjustedEnd":"0:02:23.41","start":"0:02:22.009","end":"0:02:23.41"},{"adjustedStart":"0:02:25.278","adjustedEnd":"0:02:26.213","start":"0:02:25.278","end":"0:02:26.213"}]},{"id":5,"text":"pain","confidence":0.8478,"language":"en-US","instances":[{"adjustedStart":"0:02:22.009","adjustedEnd":"0:02:23.41","start":"0:02:22.009","end":"0:02:23.41"},{"adjustedStart":"0:02:25.278","adjustedEnd":"0:02:26.213","start":"0:02:25.278","end":"0:02:26.213"},{"adjustedStart":"0:01:00.727","adjustedEnd":"0:01:01.661","start":"0:01:00.727","end":"0:01:01.661"}]},{"id":6,"text":"happiness","confidence":0.8041,"language":"en-US","instances":[{"adjustedStart":"0:03:03.583","adjustedEnd":"0:03:05.452","start":"0:03:03.583","end":"0:03:05.452"},{"adjustedStart":"0:03:05.452","adjustedEnd":"0:03:08.722","start":"0:03:05.452","end":"0:03:08.722"}]},{"id":7,"text":"trooper","confidence":0.7217,"language":"en-US","instances":[{"adjustedStart":"0:02:32.753","adjustedEnd":"0:02:33.22","start":"0:02:32.753","end":"0:02:33.22"},{"adjustedStart":"0:02:35.088","adjustedEnd":"0:02:35.555","start":"0:02:35.088","end":"0:02:35.555"},{"adjustedStart":"0:02:34.154","adjustedEnd":"0:02:34.621","start":"0:02:34.154","end":"0:02:34.621"},{"adjustedStart":"0:02:35.555","adjustedEnd":"0:02:36.022","start":"0:02:35.555","end":"0:02:36.022"},{"adjustedStart":"0:02:32.285","adjustedEnd":"0:02:32.752","start":"0:02:32.285","end":"0:02:32.752"}]},{"id":8,"text":"hope","confidence":0.69,"language":"en-US","instances":[{"adjustedStart":"0:01:00.26","adjustedEnd":"0:01:00.727","start":"0:01:00.26","end":"0:01:00.727"},{"adjustedStart":"0:00:20.554","adjustedEnd":"0:00:21.021","start":"0:00:20.554","end":"0:00:21.021"},{"adjustedStart":"0:00:28.962","adjustedEnd":"0:00:29.429","start":"0:00:28.962","end":"0:00:29.429"},{"adjustedStart":"0:01:30.624","adjustedEnd":"0:01:31.091","start":"0:01:30.624","end":"0:01:31.091"},{"adjustedStart":"0:00:33.166","adjustedEnd":"0:00:33.633","start":"0:00:33.166","end":"0:00:33.633"},{"adjustedStart":"0:01:05.399","adjustedEnd":"0:01:05.866","start":"0:01:05.399","end":"0:01:05.866"},{"adjustedStart":"0:02:08.929","adjustedEnd":"0:02:09.396","start":"0:02:08.929","end":"0:02:09.396"},{"adjustedStart":"0:00:53.72","adjustedEnd":"0:00:54.654","start":"0:00:53.72","end":"0:00:54.654"},{"adjustedStart":"0:01:07.734","adjustedEnd":"0:01:08.201","start":"0:01:07.734","end":"0:01:08.201"},{"adjustedStart":"0:00:03.27","adjustedEnd":"0:00:05.605","start":"0:00:03.27","end":"0:00:05.605"},{"adjustedStart":"0:00:07.941","adjustedEnd":"0:00:08.408","start":"0:00:07.941","end":"0:00:08.408"},{"adjustedStart":"0:00:08.408","adjustedEnd":"0:00:08.875","start":"0:00:08.408","end":"0:00:08.875"},{"adjustedStart":"0:03:16.196","adjustedEnd":"0:03:19.933","start":"0:03:16.196","end":"0:03:19.933"}]},{"id":9,"text":"broken","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:00:11.66","adjustedEnd":"0:00:21.8","start":"0:00:11.66","end":"0:00:21.8"},{"adjustedStart":"0:00:29.33","adjustedEnd":"0:00:31.95","start":"0:00:29.33","end":"0:00:31.95"},{"adjustedStart":"0:01:44.68","adjustedEnd":"0:01:53.36","start":"0:01:44.68","end":"0:01:53.36"},{"adjustedStart":"0:01:53.981","adjustedEnd":"0:01:58.185","start":"0:01:53.981","end":"0:01:58.185"}]},{"id":10,"text":"hospital","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:00:31.95","adjustedEnd":"0:00:37.25","start":"0:00:31.95","end":"0:00:37.25"},{"adjustedStart":"0:01:30.624","adjustedEnd":"0:01:34.361","start":"0:01:30.624","end":"0:01:34.361"}]},{"id":11,"text":"milo","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:02:12.6","adjustedEnd":"0:02:26.2","start":"0:02:12.6","end":"0:02:26.2"},{"adjustedStart":"0:02:32.753","adjustedEnd":"0:02:33.22","start":"0:02:32.753","end":"0:02:33.22"},{"adjustedStart":"0:02:44.898","adjustedEnd":"0:02:45.365","start":"0:02:44.898","end":"0:02:45.365"}]},{"id":12,"text":"paws","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:00:20.554","adjustedEnd":"0:00:21.021","start":"0:00:20.554","end":"0:00:21.021"},{"adjustedStart":"0:01:07.734","adjustedEnd":"0:01:08.201","start":"0:01:07.734","end":"0:01:08.201"},{"adjustedStart":"0:00:20.087","adjustedEnd":"0:00:20.554","start":"0:00:20.087","end":"0:00:20.554"},{"adjustedStart":"0:00:15.415","adjustedEnd":"0:00:15.882","start":"0:00:15.415","end":"0:00:15.882"},{"adjustedStart":"0:00:47.648","adjustedEnd":"0:00:48.115","start":"0:00:47.648","end":"0:00:48.115"},{"adjustedStart":"0:01:28.288","adjustedEnd":"0:01:28.755","start":"0:01:28.288","end":"0:01:28.755"},{"adjustedStart":"0:03:07.788","adjustedEnd":"0:03:08.255","start":"0:03:07.788","end":"0:03:08.255"},{"adjustedStart":"0:00:07.941","adjustedEnd":"0:00:08.408","start":"0:00:07.941","end":"0:00:08.408"},{"adjustedStart":"0:03:01.248","adjustedEnd":"0:03:01.715","start":"0:03:01.248","end":"0:03:01.715"},{"adjustedStart":"0:00:08.408","adjustedEnd":"0:00:08.875","start":"0:00:08.408","end":"0:00:08.875"},{"adjustedStart":"0:02:58.912","adjustedEnd":"0:03:03.116","start":"0:02:58.912","end":"0:03:03.116"}]},{"id":13,"text":"notified","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:00:08.408","adjustedEnd":"0:00:08.875","start":"0:00:08.408","end":"0:00:08.875"},{"adjustedStart":"0:00:07.474","adjustedEnd":"0:00:07.941","start":"0:00:07.474","end":"0:00:07.941"}]},{"id":14,"text":"support","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:03:12.926","adjustedEnd":"0:03:14.794","start":"0:03:12.926","end":"0:03:14.794"},{"adjustedStart":"0:03:15.262","adjustedEnd":"0:03:15.729","start":"0:03:15.262","end":"0:03:15.729"}]},{"id":15,"text":"created","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:01:05.866","adjustedEnd":"0:01:07.267","start":"0:01:05.866","end":"0:01:07.267"},{"adjustedStart":"0:01:03.53","adjustedEnd":"0:01:05.399","start":"0:01:03.53","end":"0:01:05.399"}]},{"id":16,"text":"life","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:03:03.583","adjustedEnd":"0:03:05.452","start":"0:03:03.583","end":"0:03:05.452"},{"adjustedStart":"0:03:05.452","adjustedEnd":"0:03:08.722","start":"0:03:05.452","end":"0:03:08.722"},{"adjustedStart":"0:03:16.196","adjustedEnd":"0:03:19.933","start":"0:03:16.196","end":"0:03:19.933"}]},{"id":17,"text":"found","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:02:58.912","adjustedEnd":"0:03:00.313","start":"0:02:58.912","end":"0:03:00.313"},{"adjustedStart":"0:03:00.313","adjustedEnd":"0:03:03.116","start":"0:03:00.313","end":"0:03:03.116"}]},{"id":18,"text":"antibiotics","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:02:22.009","adjustedEnd":"0:02:23.41","start":"0:02:22.009","end":"0:02:23.41"},{"adjustedStart":"0:02:25.278","adjustedEnd":"0:02:26.213","start":"0:02:25.278","end":"0:02:26.213"}]},{"id":19,"text":"determined","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:02:11.732","adjustedEnd":"0:02:12.199","start":"0:02:11.732","end":"0:02:12.199"},{"adjustedStart":"0:02:11.264","adjustedEnd":"0:02:11.731","start":"0:02:11.264","end":"0:02:11.731"},{"adjustedStart":"0:02:09.863","adjustedEnd":"0:02:10.33","start":"0:02:09.863","end":"0:02:10.33"}]},{"id":20,"text":"makeshif","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:01:05.866","adjustedEnd":"0:01:07.267","start":"0:01:05.866","end":"0:01:07.267"},{"adjustedStart":"0:01:03.53","adjustedEnd":"0:01:05.399","start":"0:01:03.53","end":"0:01:05.399"}]},{"id":21,"text":"responded","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:00:10.277","adjustedEnd":"0:00:12.145","start":"0:00:10.277","end":"0:00:12.145"},{"adjustedStart":"0:00:12.145","adjustedEnd":"0:00:12.612","start":"0:00:12.145","end":"0:00:12.612"}]},{"id":22,"text":"subscribe","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:03:27.407","adjustedEnd":"0:03:27.874","start":"0:03:27.407","end":"0:03:27.874"},{"adjustedStart":"0:00:03.27","adjustedEnd":"0:00:05.605","start":"0:00:03.27","end":"0:00:05.605"}]},{"id":23,"text":"knew","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:01:08.669","adjustedEnd":"0:01:09.603","start":"0:01:08.669","end":"0:01:09.603"},{"adjustedStart":"0:01:10.07","adjustedEnd":"0:01:11.004","start":"0:01:10.07","end":"0:01:11.004"}]},{"id":24,"text":"rescues","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:03:15.262","adjustedEnd":"0:03:15.729","start":"0:03:15.262","end":"0:03:15.729"},{"adjustedStart":"0:03:12.459","adjustedEnd":"0:03:14.794","start":"0:03:12.459","end":"0:03:14.794"}]},{"id":25,"text":"happy","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:02:32.285","adjustedEnd":"0:02:33.22","start":"0:02:32.285","end":"0:02:33.22"},{"adjustedStart":"0:02:42.095","adjustedEnd":"0:02:44.431","start":"0:02:42.095","end":"0:02:44.431"}]},{"id":26,"text":"suffer","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:02:11.732","adjustedEnd":"0:02:12.199","start":"0:02:11.732","end":"0:02:12.199"},{"adjustedStart":"0:02:10.33","adjustedEnd":"0:02:10.797","start":"0:02:10.33","end":"0:02:10.797"}]},{"id":27,"text":"yard","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:00:07.007","adjustedEnd":"0:00:07.941","start":"0:00:07.007","end":"0:00:07.941"},{"adjustedStart":"0:00:07.941","adjustedEnd":"0:00:08.408","start":"0:00:07.941","end":"0:00:08.408"},{"adjustedStart":"0:00:08.408","adjustedEnd":"0:00:08.875","start":"0:00:08.408","end":"0:00:08.875"}]},{"id":28,"text":"learly","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:00:33.634","adjustedEnd":"0:00:34.101","start":"0:00:33.634","end":"0:00:34.101"},{"adjustedStart":"0:00:31.765","adjustedEnd":"0:00:32.232","start":"0:00:31.765","end":"0:00:32.232"},{"adjustedStart":"0:00:29.429","adjustedEnd":"0:00:30.364","start":"0:00:29.429","end":"0:00:30.364"},{"adjustedStart":"0:00:30.364","adjustedEnd":"0:00:31.765","start":"0:00:30.364","end":"0:00:31.765"}]},{"id":29,"text":"cooperati","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:01:07.267","adjustedEnd":"0:01:07.734","start":"0:01:07.267","end":"0:01:07.734"},{"adjustedStart":"0:01:03.53","adjustedEnd":"0:01:05.399","start":"0:01:03.53","end":"0:01:05.399"}]},{"id":30,"text":"ulder","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:02:04.257","adjustedEnd":"0:02:04.724","start":"0:02:04.257","end":"0:02:04.724"},{"adjustedStart":"0:02:02.856","adjustedEnd":"0:02:03.323","start":"0:02:02.856","end":"0:02:03.323"}]}],"topics":[{"id":1,"name":"Standards","referenceId":"Category:Standards","referenceUrl":"https://en.wikipedia.org/wiki/Category:Standards","referenceType":"Wikipedia","confidence":0.6865,"iabName":null,"language":"en-US","instances":[{"adjustedStart":"0:02:01.922","adjustedEnd":"0:02:02.389","start":"0:02:01.922","end":"0:02:02.389"}]},{"id":2,"name":"Systems engineering","referenceId":"Category:Systems_engineering","referenceUrl":"https://en.wikipedia.org/wiki/Category:Systems_engineering","referenceType":"Wikipedia","confidence":0.686,"iabName":null,"language":"en-US","instances":[{"adjustedStart":"0:02:01.922","adjustedEnd":"0:02:02.389","start":"0:02:01.922","end":"0:02:02.389"}]},{"id":3,"name":"Television films","referenceId":"Category:Television_films","referenceUrl":"https://en.wikipedia.org/wiki/Category:Television_films","referenceType":"Wikipedia","confidence":0.6801,"iabName":null,"language":"en-US","instances":[{"adjustedStart":"0:00:33.166","adjustedEnd":"0:00:33.633","start":"0:00:33.166","end":"0:00:33.633"}]},{"id":4,"name":"Films about diseases","referenceId":"Category:Films_about_diseases","referenceUrl":"https://en.wikipedia.org/wiki/Category:Films_about_diseases","referenceType":"Wikipedia","confidence":0.6801,"iabName":null,"language":"en-US","instances":[{"adjustedStart":"0:00:33.166","adjustedEnd":"0:00:33.633","start":"0:00:33.166","end":"0:00:33.633"}]},{"id":5,"name":"Animal rescue groups","referenceId":"Category:Animal_rescue_groups","referenceUrl":"https://en.wikipedia.org/wiki/Category:Animal_rescue_groups","referenceType":"Wikipedia","confidence":0.6596,"iabName":null,"language":"en-US","instances":[{"adjustedStart":"0:00:02.803","adjustedEnd":"0:00:05.605","start":"0:00:02.803","end":"0:00:05.605"}]},{"id":6,"name":"Animal welfare organizations","referenceId":"Category:Animal_welfare_organizations","referenceUrl":"https://en.wikipedia.org/wiki/Category:Animal_welfare_organizations","referenceType":"Wikipedia","confidence":0.6596,"iabName":null,"language":"en-US","instances":[{"adjustedStart":"0:00:02.803","adjustedEnd":"0:00:05.605","start":"0:00:02.803","end":"0:00:05.605"}]},{"id":7,"name":"Police and Law Enforcement","referenceId":"Law and Justice/Justice/Police and Law Enforcement","referenceType":"VideoIndexer","iptcName":"Crime, Law and Justice/justice and rights","confidence":0.5311,"iabName":"News and Politics","language":"en-US","instances":[{"adjustedStart":"0:00:00","adjustedEnd":"0:03:32.045","start":"0:00:00","end":"0:03:32.045"}]},{"id":8,"name":"Neighbors Tv Series","referenceId":"Neighbors Tv Series","referenceType":"VideoIndexer","confidence":0.5217,"iabName":null,"language":"en-US","instances":[{"adjustedStart":"0:00:00","adjustedEnd":"0:03:32.045","start":"0:00:00","end":"0:03:32.045"}]}],"faces":[{"id":1002,"name":"Unknown #1","confidence":0,"description":null,"thumbnailId":"17821543-5fde-45b1-a4de-a338c30a4cb4","title":null,"imageUrl":null,"thumbnails":[{"id":"17821543-5fde-45b1-a4de-a338c30a4cb4","fileName":"FaceInstanceThumbnail_17821543-5fde-45b1-a4de-a338c30a4cb4.jpg","instances":[{"adjustedStart":"0:03:29.342","adjustedEnd":"0:03:29.375","start":"0:03:29.342","end":"0:03:29.375"}]}],"instances":[{"thumbnailsIds":["17821543-5fde-45b1-a4de-a338c30a4cb4"],"adjustedStart":"0:03:29.342","adjustedEnd":"0:03:31.044","start":"0:03:29.342","end":"0:03:31.044"}]},{"id":1001,"name":"Unknown #2","confidence":0,"description":null,"thumbnailId":"b6e9e256-7bd2-4d04-bc09-4a7162f9a2b3","title":null,"imageUrl":null,"thumbnails":[{"id":"b6e9e256-7bd2-4d04-bc09-4a7162f9a2b3","fileName":"FaceInstanceThumbnail_b6e9e256-7bd2-4d04-bc09-4a7162f9a2b3.jpg","instances":[{"adjustedStart":"0:02:58.311","adjustedEnd":"0:02:58.344","start":"0:02:58.311","end":"0:02:58.344"}]}],"instances":[{"thumbnailsIds":["b6e9e256-7bd2-4d04-bc09-4a7162f9a2b3"],"adjustedStart":"0:02:58.311","adjustedEnd":"0:02:59.479","start":"0:02:58.311","end":"0:02:59.479"}]}],"labels":[{"id":1,"name":"dark","referenceId":"dark","language":"en-US","instances":[{"confidence":0.9882,"adjustedStart":"0:00:00","adjustedEnd":"0:00:01.068","start":"0:00:00","end":"0:00:01.068"},{"confidence":0.925,"adjustedStart":"0:01:51.044","adjustedEnd":"0:01:54.248","start":"0:01:51.044","end":"0:01:54.248"}]},{"id":2,"name":"moon","language":"en-US","instances":[{"confidence":0.9608,"adjustedStart":"0:00:00","adjustedEnd":"0:00:01.068","start":"0:00:00","end":"0:00:01.068"}]},{"id":3,"name":"darkness","language":"en-US","instances":[{"confidence":0.9171,"adjustedStart":"0:00:00","adjustedEnd":"0:00:01.068","start":"0:00:00","end":"0:00:01.068"}]},{"id":4,"name":"colorfulness","language":"en-US","instances":[{"confidence":0.9389,"adjustedStart":"0:00:01.068","adjustedEnd":"0:00:03.203","start":"0:00:01.068","end":"0:00:03.203"},{"confidence":0.9416,"adjustedStart":"0:03:27.14","adjustedEnd":"0:03:29.276","start":"0:03:27.14","end":"0:03:29.276"}]},{"id":5,"name":"vortex","language":"en-US","instances":[{"confidence":0.9234,"adjustedStart":"0:00:01.068","adjustedEnd":"0:00:03.203","start":"0:00:01.068","end":"0:00:03.203"},{"confidence":0.9462,"adjustedStart":"0:03:27.14","adjustedEnd":"0:03:28.208","start":"0:03:27.14","end":"0:03:28.208"}]},{"id":6,"name":"light","referenceId":"physical phenomenon/energy/radiation/light","language":"en-US","instances":[{"confidence":0.9036,"adjustedStart":"0:00:01.068","adjustedEnd":"0:00:02.136","start":"0:00:01.068","end":"0:00:02.136"},{"confidence":0.9853,"adjustedStart":"0:03:27.14","adjustedEnd":"0:03:28.208","start":"0:03:27.14","end":"0:03:28.208"}]},{"id":7,"name":"electric blue","language":"en-US","instances":[{"confidence":0.901,"adjustedStart":"0:00:01.068","adjustedEnd":"0:00:02.136","start":"0:00:01.068","end":"0:00:02.136"},{"confidence":0.9178,"adjustedStart":"0:03:27.14","adjustedEnd":"0:03:28.208","start":"0:03:27.14","end":"0:03:28.208"}]},{"id":8,"name":"cup","referenceId":"tableware/cup","language":"en-US","instances":[{"confidence":0.8519,"adjustedStart":"0:00:01.068","adjustedEnd":"0:00:02.136","start":"0:00:01.068","end":"0:00:02.136"}]},{"id":9,"name":"text","referenceId":"communication/writing/text","language":"en-US","instances":[{"confidence":0.9999,"adjustedStart":"0:00:02.135","adjustedEnd":"0:00:09.61","start":"0:00:02.135","end":"0:00:09.61"},{"confidence":0.9985,"adjustedStart":"0:00:10.677","adjustedEnd":"0:00:13.881","start":"0:00:10.677","end":"0:00:13.881"},{"confidence":0.9181,"adjustedStart":"0:00:29.897","adjustedEnd":"0:00:32.032","start":"0:00:29.897","end":"0:00:32.032"},{"confidence":0.9986,"adjustedStart":"0:00:59.793","adjustedEnd":"0:01:02.997","start":"0:00:59.793","end":"0:01:02.997"},{"confidence":0.9934,"adjustedStart":"0:01:04.064","adjustedEnd":"0:01:08.335","start":"0:01:04.064","end":"0:01:08.335"},{"confidence":0.989,"adjustedStart":"0:01:10.47","adjustedEnd":"0:01:12.606","start":"0:01:10.47","end":"0:01:12.606"},{"confidence":0.9776,"adjustedStart":"0:01:17.945","adjustedEnd":"0:01:19.013","start":"0:01:17.945","end":"0:01:19.013"},{"confidence":0.9226,"adjustedStart":"0:01:25.419","adjustedEnd":"0:01:26.487","start":"0:01:25.419","end":"0:01:26.487"},{"confidence":0.9957,"adjustedStart":"0:01:30.757","adjustedEnd":"0:01:35.029","start":"0:01:30.757","end":"0:01:35.029"},{"confidence":0.9985,"adjustedStart":"0:01:51.044","adjustedEnd":"0:02:00.654","start":"0:01:51.044","end":"0:02:00.654"},{"confidence":0.9372,"adjustedStart":"0:02:01.722","adjustedEnd":"0:02:05.993","start":"0:02:01.722","end":"0:02:05.993"},{"confidence":0.9523,"adjustedStart":"0:02:09.196","adjustedEnd":"0:02:12.399","start":"0:02:09.196","end":"0:02:12.399"},{"confidence":0.9991,"adjustedStart":"0:02:32.686","adjustedEnd":"0:02:38.025","start":"0:02:32.686","end":"0:02:38.025"},{"confidence":0.9989,"adjustedStart":"0:02:42.295","adjustedEnd":"0:02:47.634","start":"0:02:42.295","end":"0:02:47.634"},{"confidence":0.9957,"adjustedStart":"0:02:59.379","adjustedEnd":"0:03:08.989","start":"0:02:59.379","end":"0:03:08.989"},{"confidence":0.9859,"adjustedStart":"0:03:13.26","adjustedEnd":"0:03:19.666","start":"0:03:13.26","end":"0:03:19.666"},{"confidence":0.9835,"adjustedStart":"0:03:27.14","adjustedEnd":"0:03:31.978","start":"0:03:27.14","end":"0:03:31.978"}]},{"id":10,"name":"purple","referenceId":"color/purple","language":"en-US","instances":[{"confidence":0.9558,"adjustedStart":"0:00:02.135","adjustedEnd":"0:00:03.203","start":"0:00:02.135","end":"0:00:03.203"},{"confidence":0.952,"adjustedStart":"0:03:27.14","adjustedEnd":"0:03:29.276","start":"0:03:27.14","end":"0:03:29.276"}]},{"id":11,"name":"circle","language":"en-US","instances":[{"confidence":0.9376,"adjustedStart":"0:00:02.135","adjustedEnd":"0:00:03.203","start":"0:00:02.135","end":"0:00:03.203"}]},{"id":12,"name":"violet","language":"en-US","instances":[{"confidence":0.9281,"adjustedStart":"0:00:02.135","adjustedEnd":"0:00:03.203","start":"0:00:02.135","end":"0:00:03.203"},{"confidence":0.9273,"adjustedStart":"0:03:28.208","adjustedEnd":"0:03:29.276","start":"0:03:28.208","end":"0:03:29.276"}]},{"id":13,"name":"magenta","language":"en-US","instances":[{"confidence":0.9038,"adjustedStart":"0:00:02.135","adjustedEnd":"0:00:03.203","start":"0:00:02.135","end":"0:00:03.203"},{"confidence":0.9112,"adjustedStart":"0:03:28.208","adjustedEnd":"0:03:29.276","start":"0:03:28.208","end":"0:03:29.276"}]},{"id":14,"name":"dog","referenceId":"animal/mammal/dog","language":"en-US","instances":[{"confidence":0.9392,"adjustedStart":"0:00:03.203","adjustedEnd":"0:00:04.271","start":"0:00:03.203","end":"0:00:04.271"},{"confidence":0.9511,"adjustedStart":"0:00:23.49","adjustedEnd":"0:00:53.387","start":"0:00:23.49","end":"0:00:53.387"},{"confidence":0.9623,"adjustedStart":"0:00:54.454","adjustedEnd":"0:00:59.793","start":"0:00:54.454","end":"0:00:59.793"},{"confidence":0.9368,"adjustedStart":"0:01:00.861","adjustedEnd":"0:01:02.997","start":"0:01:00.861","end":"0:01:02.997"},{"confidence":0.9383,"adjustedStart":"0:01:05.132","adjustedEnd":"0:01:06.2","start":"0:01:05.132","end":"0:01:06.2"},{"confidence":0.9364,"adjustedStart":"0:01:22.215","adjustedEnd":"0:01:23.283","start":"0:01:22.215","end":"0:01:23.283"},{"confidence":0.8646,"adjustedStart":"0:01:24.351","adjustedEnd":"0:01:25.419","start":"0:01:24.351","end":"0:01:25.419"},{"confidence":0.9228,"adjustedStart":"0:01:27.554","adjustedEnd":"0:01:29.69","start":"0:01:27.554","end":"0:01:29.69"},{"confidence":0.9705,"adjustedStart":"0:01:30.757","adjustedEnd":"0:01:43.57","start":"0:01:30.757","end":"0:01:43.57"},{"confidence":0.9694,"adjustedStart":"0:02:01.722","adjustedEnd":"0:02:02.79","start":"0:02:01.722","end":"0:02:02.79"},{"confidence":0.933,"adjustedStart":"0:02:15.602","adjustedEnd":"0:02:16.67","start":"0:02:15.602","end":"0:02:16.67"},{"confidence":0.9908,"adjustedStart":"0:02:18.805","adjustedEnd":"0:02:29.483","start":"0:02:18.805","end":"0:02:29.483"},{"confidence":0.9752,"adjustedStart":"0:02:30.55","adjustedEnd":"0:02:31.618","start":"0:02:30.55","end":"0:02:31.618"},{"confidence":0.8725,"adjustedStart":"0:02:32.686","adjustedEnd":"0:02:34.822","start":"0:02:32.686","end":"0:02:34.822"},{"confidence":0.9679,"adjustedStart":"0:02:39.092","adjustedEnd":"0:02:43.363","start":"0:02:39.092","end":"0:02:43.363"},{"confidence":0.9316,"adjustedStart":"0:02:46.566","adjustedEnd":"0:02:47.634","start":"0:02:46.566","end":"0:02:47.634"},{"confidence":0.9335,"adjustedStart":"0:02:50.837","adjustedEnd":"0:02:55.109","start":"0:02:50.837","end":"0:02:55.109"},{"confidence":0.9598,"adjustedStart":"0:02:56.176","adjustedEnd":"0:03:02.583","start":"0:02:56.176","end":"0:03:02.583"},{"confidence":0.9577,"adjustedStart":"0:03:03.65","adjustedEnd":"0:03:27.141","start":"0:03:03.65","end":"0:03:27.141"},{"confidence":0.9397,"adjustedStart":"0:03:29.276","adjustedEnd":"0:03:30.344","start":"0:03:29.276","end":"0:03:30.344"}]},{"id":15,"name":"animal","referenceId":"organism/animal","language":"en-US","instances":[{"confidence":0.9354,"adjustedStart":"0:00:03.203","adjustedEnd":"0:00:04.271","start":"0:00:03.203","end":"0:00:04.271"},{"confidence":0.9292,"adjustedStart":"0:00:22.422","adjustedEnd":"0:00:26.694","start":"0:00:22.422","end":"0:00:26.694"},{"confidence":0.898,"adjustedStart":"0:00:27.761","adjustedEnd":"0:00:36.303","start":"0:00:27.761","end":"0:00:36.303"},{"confidence":0.9408,"adjustedStart":"0:00:37.371","adjustedEnd":"0:00:42.71","start":"0:00:37.371","end":"0:00:42.71"},{"confidence":0.9441,"adjustedStart":"0:00:43.777","adjustedEnd":"0:00:44.845","start":"0:00:43.777","end":"0:00:44.845"},{"confidence":0.9874,"adjustedStart":"0:00:45.913","adjustedEnd":"0:00:48.048","start":"0:00:45.913","end":"0:00:48.048"},{"confidence":0.9194,"adjustedStart":"0:00:49.116","adjustedEnd":"0:00:53.387","start":"0:00:49.116","end":"0:00:53.387"},{"confidence":0.9227,"adjustedStart":"0:00:54.454","adjustedEnd":"0:01:00.861","start":"0:00:54.454","end":"0:01:00.861"},{"confidence":0.965,"adjustedStart":"0:01:01.929","adjustedEnd":"0:01:02.997","start":"0:01:01.929","end":"0:01:02.997"},{"confidence":0.9573,"adjustedStart":"0:01:05.132","adjustedEnd":"0:01:06.2","start":"0:01:05.132","end":"0:01:06.2"},{"confidence":0.9885,"adjustedStart":"0:01:24.351","adjustedEnd":"0:01:25.419","start":"0:01:24.351","end":"0:01:25.419"},{"confidence":0.9845,"adjustedStart":"0:01:28.622","adjustedEnd":"0:01:29.69","start":"0:01:28.622","end":"0:01:29.69"},{"confidence":0.9394,"adjustedStart":"0:01:30.757","adjustedEnd":"0:01:38.232","start":"0:01:30.757","end":"0:01:38.232"},{"confidence":0.9533,"adjustedStart":"0:01:39.299","adjustedEnd":"0:01:43.57","start":"0:01:39.299","end":"0:01:43.57"},{"confidence":0.9746,"adjustedStart":"0:02:01.722","adjustedEnd":"0:02:02.79","start":"0:02:01.722","end":"0:02:02.79"},{"confidence":0.9699,"adjustedStart":"0:02:18.805","adjustedEnd":"0:02:28.415","start":"0:02:18.805","end":"0:02:28.415"},{"confidence":0.9565,"adjustedStart":"0:02:30.55","adjustedEnd":"0:02:31.618","start":"0:02:30.55","end":"0:02:31.618"},{"confidence":0.9646,"adjustedStart":"0:02:33.754","adjustedEnd":"0:02:34.822","start":"0:02:33.754","end":"0:02:34.822"},{"confidence":0.9389,"adjustedStart":"0:02:39.092","adjustedEnd":"0:02:43.363","start":"0:02:39.092","end":"0:02:43.363"},{"confidence":0.9127,"adjustedStart":"0:02:50.837","adjustedEnd":"0:02:55.109","start":"0:02:50.837","end":"0:02:55.109"},{"confidence":0.9682,"adjustedStart":"0:02:56.176","adjustedEnd":"0:02:57.244","start":"0:02:56.176","end":"0:02:57.244"},{"confidence":0.9421,"adjustedStart":"0:02:58.311","adjustedEnd":"0:03:02.583","start":"0:02:58.311","end":"0:03:02.583"},{"confidence":0.9429,"adjustedStart":"0:03:04.718","adjustedEnd":"0:03:11.125","start":"0:03:04.718","end":"0:03:11.125"},{"confidence":0.9214,"adjustedStart":"0:03:12.192","adjustedEnd":"0:03:25.005","start":"0:03:12.192","end":"0:03:25.005"},{"confidence":0.9509,"adjustedStart":"0:03:29.276","adjustedEnd":"0:03:30.344","start":"0:03:29.276","end":"0:03:30.344"}]},{"id":16,"name":"graphic","language":"en-US","instances":[{"confidence":0.9866,"adjustedStart":"0:00:04.271","adjustedEnd":"0:00:05.339","start":"0:00:04.271","end":"0:00:05.339"},{"confidence":0.9901,"adjustedStart":"0:01:54.247","adjustedEnd":"0:01:55.315","start":"0:01:54.247","end":"0:01:55.315"},{"confidence":0.9884,"adjustedStart":"0:03:30.343","adjustedEnd":"0:03:31.978","start":"0:03:30.343","end":"0:03:31.978"}]},{"id":17,"name":"screenshot","language":"en-US","instances":[{"confidence":0.9426,"adjustedStart":"0:00:05.339","adjustedEnd":"0:00:12.813","start":"0:00:05.339","end":"0:00:12.813"},{"confidence":0.9,"adjustedStart":"0:00:16.016","adjustedEnd":"0:00:17.084","start":"0:00:16.016","end":"0:00:17.084"},{"confidence":0.9304,"adjustedStart":"0:01:02.996","adjustedEnd":"0:01:05.132","start":"0:01:02.996","end":"0:01:05.132"},{"confidence":0.9573,"adjustedStart":"0:01:06.199","adjustedEnd":"0:01:08.335","start":"0:01:06.199","end":"0:01:08.335"},{"confidence":0.9588,"adjustedStart":"0:01:09.403","adjustedEnd":"0:01:13.674","start":"0:01:09.403","end":"0:01:13.674"},{"confidence":0.9283,"adjustedStart":"0:01:25.419","adjustedEnd":"0:01:28.622","start":"0:01:25.419","end":"0:01:28.622"},{"confidence":0.9515,"adjustedStart":"0:01:29.69","adjustedEnd":"0:01:30.758","start":"0:01:29.69","end":"0:01:30.758"},{"confidence":0.9056,"adjustedStart":"0:01:47.841","adjustedEnd":"0:01:48.909","start":"0:01:47.841","end":"0:01:48.909"},{"confidence":0.9839,"adjustedStart":"0:01:51.044","adjustedEnd":"0:01:58.519","start":"0:01:51.044","end":"0:01:58.519"},{"confidence":0.9524,"adjustedStart":"0:01:59.586","adjustedEnd":"0:02:00.654","start":"0:01:59.586","end":"0:02:00.654"},{"confidence":0.9008,"adjustedStart":"0:02:32.686","adjustedEnd":"0:02:33.754","start":"0:02:32.686","end":"0:02:33.754"},{"confidence":0.9245,"adjustedStart":"0:02:34.821","adjustedEnd":"0:02:38.025","start":"0:02:34.821","end":"0:02:38.025"},{"confidence":0.9312,"adjustedStart":"0:02:44.431","adjustedEnd":"0:02:45.499","start":"0:02:44.431","end":"0:02:45.499"},{"confidence":0.9038,"adjustedStart":"0:02:46.566","adjustedEnd":"0:02:47.634","start":"0:02:46.566","end":"0:02:47.634"},{"confidence":0.9132,"adjustedStart":"0:03:02.582","adjustedEnd":"0:03:06.854","start":"0:03:02.582","end":"0:03:06.854"},{"confidence":0.9383,"adjustedStart":"0:03:13.26","adjustedEnd":"0:03:14.328","start":"0:03:13.26","end":"0:03:14.328"},{"confidence":0.9126,"adjustedStart":"0:03:31.411","adjustedEnd":"0:03:31.978","start":"0:03:31.411","end":"0:03:31.978"}]},{"id":18,"name":"outdoor","language":"en-US","instances":[{"confidence":0.9897,"adjustedStart":"0:00:06.406","adjustedEnd":"0:00:09.61","start":"0:00:06.406","end":"0:00:09.61"},{"confidence":0.8029,"adjustedStart":"0:00:10.677","adjustedEnd":"0:00:11.745","start":"0:00:10.677","end":"0:00:11.745"},{"confidence":0.8798,"adjustedStart":"0:00:12.813","adjustedEnd":"0:00:16.016","start":"0:00:12.813","end":"0:00:16.016"},{"confidence":0.9475,"adjustedStart":"0:00:17.084","adjustedEnd":"0:00:36.303","start":"0:00:17.084","end":"0:00:36.303"},{"confidence":0.8781,"adjustedStart":"0:00:38.438","adjustedEnd":"0:00:40.574","start":"0:00:38.438","end":"0:00:40.574"},{"confidence":0.8482,"adjustedStart":"0:00:41.642","adjustedEnd":"0:00:42.71","start":"0:00:41.642","end":"0:00:42.71"},{"confidence":0.921,"adjustedStart":"0:00:44.845","adjustedEnd":"0:00:51.251","start":"0:00:44.845","end":"0:00:51.251"},{"confidence":0.8002,"adjustedStart":"0:01:06.199","adjustedEnd":"0:01:07.267","start":"0:01:06.199","end":"0:01:07.267"},{"confidence":0.9356,"adjustedStart":"0:01:08.335","adjustedEnd":"0:01:09.403","start":"0:01:08.335","end":"0:01:09.403"},{"confidence":0.9178,"adjustedStart":"0:01:13.674","adjustedEnd":"0:01:14.742","start":"0:01:13.674","end":"0:01:14.742"},{"confidence":0.9681,"adjustedStart":"0:01:16.877","adjustedEnd":"0:01:17.945","start":"0:01:16.877","end":"0:01:17.945"},{"confidence":0.8702,"adjustedStart":"0:01:19.012","adjustedEnd":"0:01:20.08","start":"0:01:19.012","end":"0:01:20.08"},{"confidence":0.8205,"adjustedStart":"0:01:29.69","adjustedEnd":"0:01:30.758","start":"0:01:29.69","end":"0:01:30.758"},{"confidence":0.8789,"adjustedStart":"0:01:36.096","adjustedEnd":"0:01:38.232","start":"0:01:36.096","end":"0:01:38.232"}]},{"id":19,"name":"tree","referenceId":"plant/tree","language":"en-US","instances":[{"confidence":0.9594,"adjustedStart":"0:00:06.406","adjustedEnd":"0:00:08.542","start":"0:00:06.406","end":"0:00:08.542"}]},{"id":20,"name":"cloud","referenceId":"natural phenomenon/weather/cloud","language":"en-US","instances":[{"confidence":0.9323,"adjustedStart":"0:00:06.406","adjustedEnd":"0:00:09.61","start":"0:00:06.406","end":"0:00:09.61"}]},{"id":21,"name":"sign","referenceId":"communication/indication/sign","language":"en-US","instances":[{"confidence":0.9351,"adjustedStart":"0:00:06.406","adjustedEnd":"0:00:07.474","start":"0:00:06.406","end":"0:00:07.474"}]},{"id":22,"name":"sky","referenceId":"atmosphere/sky","language":"en-US","instances":[{"confidence":0.9331,"adjustedStart":"0:00:06.406","adjustedEnd":"0:00:09.61","start":"0:00:06.406","end":"0:00:09.61"}]},{"id":23,"name":"car","referenceId":"conveyance/vehicle/car","language":"en-US","instances":[{"confidence":0.9771,"adjustedStart":"0:00:08.542","adjustedEnd":"0:00:09.61","start":"0:00:08.542","end":"0:00:09.61"},{"confidence":0.9082,"adjustedStart":"0:01:16.877","adjustedEnd":"0:01:17.945","start":"0:01:16.877","end":"0:01:17.945"}]},{"id":24,"name":"vehicle","referenceId":"instrumentality/conveyance/vehicle","language":"en-US","instances":[{"confidence":0.9725,"adjustedStart":"0:00:08.542","adjustedEnd":"0:00:09.61","start":"0:00:08.542","end":"0:00:09.61"},{"confidence":0.9458,"adjustedStart":"0:01:30.757","adjustedEnd":"0:01:31.825","start":"0:01:30.757","end":"0:01:31.825"},{"confidence":0.9468,"adjustedStart":"0:01:33.961","adjustedEnd":"0:01:35.029","start":"0:01:33.961","end":"0:01:35.029"},{"confidence":0.9646,"adjustedStart":"0:01:36.096","adjustedEnd":"0:01:38.232","start":"0:01:36.096","end":"0:01:38.232"}]},{"id":25,"name":"land vehicle","language":"en-US","instances":[{"confidence":0.971,"adjustedStart":"0:00:08.542","adjustedEnd":"0:00:09.61","start":"0:00:08.542","end":"0:00:09.61"}]},{"id":26,"name":"billboard","language":"en-US","instances":[{"confidence":0.904,"adjustedStart":"0:00:08.542","adjustedEnd":"0:00:09.61","start":"0:00:08.542","end":"0:00:09.61"}]},{"id":27,"name":"house","referenceId":"structure/building/house","language":"en-US","instances":[{"confidence":0.8738,"adjustedStart":"0:00:10.677","adjustedEnd":"0:00:11.745","start":"0:00:10.677","end":"0:00:11.745"}]},{"id":28,"name":"wire fencing","language":"en-US","instances":[{"confidence":0.9396,"adjustedStart":"0:00:12.813","adjustedEnd":"0:00:14.949","start":"0:00:12.813","end":"0:00:14.949"}]},{"id":29,"name":"mesh","language":"en-US","instances":[{"confidence":0.938,"adjustedStart":"0:00:12.813","adjustedEnd":"0:00:13.881","start":"0:00:12.813","end":"0:00:13.881"}]},{"id":30,"name":"chain-link fencing","language":"en-US","instances":[{"confidence":0.9158,"adjustedStart":"0:00:12.813","adjustedEnd":"0:00:14.949","start":"0:00:12.813","end":"0:00:14.949"}]},{"id":31,"name":"person","referenceId":"person","language":"en-US","instances":[{"confidence":0.9907,"adjustedStart":"0:00:14.948","adjustedEnd":"0:00:16.016","start":"0:00:14.948","end":"0:00:16.016"},{"confidence":0.9373,"adjustedStart":"0:00:29.897","adjustedEnd":"0:00:30.965","start":"0:00:29.897","end":"0:00:30.965"},{"confidence":0.9522,"adjustedStart":"0:00:33.1","adjustedEnd":"0:00:37.371","start":"0:00:33.1","end":"0:00:37.371"},{"confidence":0.9728,"adjustedStart":"0:00:42.709","adjustedEnd":"0:00:49.116","start":"0:00:42.709","end":"0:00:49.116"},{"confidence":0.9407,"adjustedStart":"0:00:51.251","adjustedEnd":"0:00:52.319","start":"0:00:51.251","end":"0:00:52.319"},{"confidence":0.9965,"adjustedStart":"0:01:08.335","adjustedEnd":"0:01:09.403","start":"0:01:08.335","end":"0:01:09.403"},{"confidence":0.9747,"adjustedStart":"0:01:10.47","adjustedEnd":"0:01:15.809","start":"0:01:10.47","end":"0:01:15.809"},{"confidence":0.9859,"adjustedStart":"0:01:16.877","adjustedEnd":"0:01:21.148","start":"0:01:16.877","end":"0:01:21.148"},{"confidence":0.9699,"adjustedStart":"0:01:43.57","adjustedEnd":"0:01:47.841","start":"0:01:43.57","end":"0:01:47.841"},{"confidence":0.9546,"adjustedStart":"0:01:48.909","adjustedEnd":"0:01:51.045","start":"0:01:48.909","end":"0:01:51.045"},{"confidence":0.9757,"adjustedStart":"0:01:56.383","adjustedEnd":"0:01:57.451","start":"0:01:56.383","end":"0:01:57.451"},{"confidence":0.9568,"adjustedStart":"0:02:00.654","adjustedEnd":"0:02:04.925","start":"0:02:00.654","end":"0:02:04.925"},{"confidence":0.9452,"adjustedStart":"0:02:07.06","adjustedEnd":"0:02:08.128","start":"0:02:07.06","end":"0:02:08.128"},{"confidence":0.9402,"adjustedStart":"0:02:09.196","adjustedEnd":"0:02:12.399","start":"0:02:09.196","end":"0:02:12.399"},{"confidence":0.9306,"adjustedStart":"0:02:13.467","adjustedEnd":"0:02:14.535","start":"0:02:13.467","end":"0:02:14.535"},{"confidence":0.9799,"adjustedStart":"0:02:15.602","adjustedEnd":"0:02:17.738","start":"0:02:15.602","end":"0:02:17.738"},{"confidence":0.9588,"adjustedStart":"0:02:34.821","adjustedEnd":"0:02:39.093","start":"0:02:34.821","end":"0:02:39.093"},{"confidence":0.9654,"adjustedStart":"0:02:42.295","adjustedEnd":"0:02:44.431","start":"0:02:42.295","end":"0:02:44.431"},{"confidence":0.9736,"adjustedStart":"0:02:47.634","adjustedEnd":"0:02:50.838","start":"0:02:47.634","end":"0:02:50.838"},{"confidence":0.928,"adjustedStart":"0:02:52.973","adjustedEnd":"0:02:54.041","start":"0:02:52.973","end":"0:02:54.041"},{"confidence":0.9508,"adjustedStart":"0:02:58.311","adjustedEnd":"0:02:59.379","start":"0:02:58.311","end":"0:02:59.379"},{"confidence":0.9756,"adjustedStart":"0:03:02.582","adjustedEnd":"0:03:04.718","start":"0:03:02.582","end":"0:03:04.718"},{"confidence":0.954,"adjustedStart":"0:03:13.26","adjustedEnd":"0:03:15.395","start":"0:03:13.26","end":"0:03:15.395"}]},{"id":32,"name":"player","referenceId":"person/contestant/player","language":"en-US","instances":[{"confidence":0.9746,"adjustedStart":"0:00:14.948","adjustedEnd":"0:00:16.016","start":"0:00:14.948","end":"0:00:16.016"}]},{"id":33,"name":"tennis","referenceId":"activity/game/tennis","language":"en-US","instances":[{"confidence":0.969,"adjustedStart":"0:00:14.948","adjustedEnd":"0:00:16.016","start":"0:00:14.948","end":"0:00:16.016"}]},{"id":34,"name":"clothing","referenceId":"consumer goods/clothing","language":"en-US","instances":[{"confidence":0.917,"adjustedStart":"0:00:14.948","adjustedEnd":"0:00:16.016","start":"0:00:14.948","end":"0:00:16.016"},{"confidence":0.9484,"adjustedStart":"0:00:30.964","adjustedEnd":"0:00:32.032","start":"0:00:30.964","end":"0:00:32.032"},{"confidence":0.9255,"adjustedStart":"0:00:33.1","adjustedEnd":"0:00:36.303","start":"0:00:33.1","end":"0:00:36.303"},{"confidence":0.9154,"adjustedStart":"0:00:48.048","adjustedEnd":"0:00:49.116","start":"0:00:48.048","end":"0:00:49.116"},{"confidence":0.9284,"adjustedStart":"0:01:08.335","adjustedEnd":"0:01:09.403","start":"0:01:08.335","end":"0:01:09.403"},{"confidence":0.9718,"adjustedStart":"0:01:13.674","adjustedEnd":"0:01:14.742","start":"0:01:13.674","end":"0:01:14.742"},{"confidence":0.9205,"adjustedStart":"0:01:17.945","adjustedEnd":"0:01:19.013","start":"0:01:17.945","end":"0:01:19.013"},{"confidence":0.9113,"adjustedStart":"0:01:43.57","adjustedEnd":"0:01:44.638","start":"0:01:43.57","end":"0:01:44.638"},{"confidence":0.9295,"adjustedStart":"0:01:48.909","adjustedEnd":"0:01:49.977","start":"0:01:48.909","end":"0:01:49.977"},{"confidence":0.9554,"adjustedStart":"0:01:56.383","adjustedEnd":"0:01:57.451","start":"0:01:56.383","end":"0:01:57.451"}]},{"id":35,"name":"toilet","referenceId":"room/toilet","language":"en-US","instances":[{"confidence":0.8605,"adjustedStart":"0:00:16.016","adjustedEnd":"0:00:17.084","start":"0:00:16.016","end":"0:00:17.084"}]},{"id":36,"name":"waste container","language":"en-US","instances":[{"confidence":0.9894,"adjustedStart":"0:00:17.084","adjustedEnd":"0:00:19.219","start":"0:00:17.084","end":"0:00:19.219"},{"confidence":0.9419,"adjustedStart":"0:00:39.506","adjustedEnd":"0:00:41.642","start":"0:00:39.506","end":"0:00:41.642"},{"confidence":0.9735,"adjustedStart":"0:00:48.048","adjustedEnd":"0:00:49.116","start":"0:00:48.048","end":"0:00:49.116"},{"confidence":0.9444,"adjustedStart":"0:00:56.59","adjustedEnd":"0:00:57.658","start":"0:00:56.59","end":"0:00:57.658"}]},{"id":37,"name":"bin","referenceId":"instrumentality/container/bin","language":"en-US","instances":[{"confidence":0.978,"adjustedStart":"0:00:17.084","adjustedEnd":"0:00:18.152","start":"0:00:17.084","end":"0:00:18.152"}]},{"id":38,"name":"container","referenceId":"instrumentality/container","language":"en-US","instances":[{"confidence":0.9721,"adjustedStart":"0:00:17.084","adjustedEnd":"0:00:18.152","start":"0:00:17.084","end":"0:00:18.152"},{"confidence":0.9488,"adjustedStart":"0:00:39.506","adjustedEnd":"0:00:40.574","start":"0:00:39.506","end":"0:00:40.574"}]},{"id":39,"name":"waste containment","language":"en-US","instances":[{"confidence":0.9552,"adjustedStart":"0:00:17.084","adjustedEnd":"0:00:18.152","start":"0:00:17.084","end":"0:00:18.152"}]},{"id":40,"name":"sidewalk","referenceId":"sidewalk","language":"en-US","instances":[{"confidence":0.9202,"adjustedStart":"0:00:17.084","adjustedEnd":"0:00:19.219","start":"0:00:17.084","end":"0:00:19.219"},{"confidence":0.9067,"adjustedStart":"0:00:20.287","adjustedEnd":"0:00:21.355","start":"0:00:20.287","end":"0:00:21.355"},{"confidence":0.9423,"adjustedStart":"0:00:34.167","adjustedEnd":"0:00:36.303","start":"0:00:34.167","end":"0:00:36.303"}]},{"id":41,"name":"recycling bin","language":"en-US","instances":[{"confidence":0.9238,"adjustedStart":"0:00:17.084","adjustedEnd":"0:00:18.152","start":"0:00:17.084","end":"0:00:18.152"}]},{"id":42,"name":"street","referenceId":"way/road/street","language":"en-US","instances":[{"confidence":0.9118,"adjustedStart":"0:00:17.084","adjustedEnd":"0:00:18.152","start":"0:00:17.084","end":"0:00:18.152"},{"confidence":0.8548,"adjustedStart":"0:00:22.422","adjustedEnd":"0:00:23.49","start":"0:00:22.422","end":"0:00:23.49"}]},{"id":43,"name":"ground","referenceId":"land/ground","language":"en-US","instances":[{"confidence":0.9034,"adjustedStart":"0:00:17.084","adjustedEnd":"0:00:19.219","start":"0:00:17.084","end":"0:00:19.219"},{"confidence":0.9025,"adjustedStart":"0:00:20.287","adjustedEnd":"0:00:21.355","start":"0:00:20.287","end":"0:00:21.355"},{"confidence":0.9429,"adjustedStart":"0:00:22.422","adjustedEnd":"0:00:26.694","start":"0:00:22.422","end":"0:00:26.694"},{"confidence":0.954,"adjustedStart":"0:00:27.761","adjustedEnd":"0:00:29.897","start":"0:00:27.761","end":"0:00:29.897"},{"confidence":0.9464,"adjustedStart":"0:00:32.032","adjustedEnd":"0:00:37.371","start":"0:00:32.032","end":"0:00:37.371"},{"confidence":0.94,"adjustedStart":"0:00:39.506","adjustedEnd":"0:00:40.574","start":"0:00:39.506","end":"0:00:40.574"},{"confidence":0.9325,"adjustedStart":"0:00:41.642","adjustedEnd":"0:00:42.71","start":"0:00:41.642","end":"0:00:42.71"},{"confidence":0.9649,"adjustedStart":"0:00:44.845","adjustedEnd":"0:00:50.184","start":"0:00:44.845","end":"0:00:50.184"}]},{"id":44,"name":"concrete","referenceId":"building material/concrete","language":"en-US","instances":[{"confidence":0.9166,"adjustedStart":"0:00:18.151","adjustedEnd":"0:00:21.355","start":"0:00:18.151","end":"0:00:21.355"},{"confidence":0.9365,"adjustedStart":"0:00:25.626","adjustedEnd":"0:00:26.694","start":"0:00:25.626","end":"0:00:26.694"},{"confidence":0.9047,"adjustedStart":"0:00:34.167","adjustedEnd":"0:00:35.235","start":"0:00:34.167","end":"0:00:35.235"}]},{"id":45,"name":"wall","referenceId":"structure/wall","language":"en-US","instances":[{"confidence":0.862,"adjustedStart":"0:00:18.151","adjustedEnd":"0:00:20.287","start":"0:00:18.151","end":"0:00:20.287"},{"confidence":0.9015,"adjustedStart":"0:01:43.57","adjustedEnd":"0:01:45.706","start":"0:01:43.57","end":"0:01:45.706"},{"confidence":0.8981,"adjustedStart":"0:02:14.534","adjustedEnd":"0:02:15.602","start":"0:02:14.534","end":"0:02:15.602"},{"confidence":0.9215,"adjustedStart":"0:02:47.634","adjustedEnd":"0:02:48.702","start":"0:02:47.634","end":"0:02:48.702"},{"confidence":0.911,"adjustedStart":"0:02:52.973","adjustedEnd":"0:02:54.041","start":"0:02:52.973","end":"0:02:54.041"}]},{"id":46,"name":"composite material","language":"en-US","instances":[{"confidence":0.9215,"adjustedStart":"0:00:19.219","adjustedEnd":"0:00:20.287","start":"0:00:19.219","end":"0:00:20.287"}]},{"id":47,"name":"floor","referenceId":"surface/floor","language":"en-US","instances":[{"confidence":0.853,"adjustedStart":"0:00:19.219","adjustedEnd":"0:00:21.355","start":"0:00:19.219","end":"0:00:21.355"},{"confidence":0.9413,"adjustedStart":"0:02:59.379","adjustedEnd":"0:03:00.447","start":"0:02:59.379","end":"0:03:00.447"},{"confidence":0.9685,"adjustedStart":"0:03:08.989","adjustedEnd":"0:03:12.192","start":"0:03:08.989","end":"0:03:12.192"},{"confidence":0.9615,"adjustedStart":"0:03:19.666","adjustedEnd":"0:03:27.141","start":"0:03:19.666","end":"0:03:27.141"}]},{"id":48,"name":"cement","referenceId":"paving/concrete/cement","language":"en-US","instances":[{"confidence":0.9014,"adjustedStart":"0:00:20.287","adjustedEnd":"0:00:21.355","start":"0:00:20.287","end":"0:00:21.355"},{"confidence":0.9034,"adjustedStart":"0:00:24.558","adjustedEnd":"0:00:26.694","start":"0:00:24.558","end":"0:00:26.694"},{"confidence":0.9062,"adjustedStart":"0:00:34.167","adjustedEnd":"0:00:35.235","start":"0:00:34.167","end":"0:00:35.235"}]},{"id":49,"name":"mammal","referenceId":"animal/mammal","language":"en-US","instances":[{"confidence":0.8961,"adjustedStart":"0:00:22.422","adjustedEnd":"0:00:26.694","start":"0:00:22.422","end":"0:00:26.694"},{"confidence":0.8987,"adjustedStart":"0:00:27.761","adjustedEnd":"0:00:30.965","start":"0:00:27.761","end":"0:00:30.965"},{"confidence":0.9196,"adjustedStart":"0:00:33.1","adjustedEnd":"0:00:36.303","start":"0:00:33.1","end":"0:00:36.303"},{"confidence":0.9189,"adjustedStart":"0:00:38.438","adjustedEnd":"0:00:39.506","start":"0:00:38.438","end":"0:00:39.506"},{"confidence":0.8466,"adjustedStart":"0:00:40.574","adjustedEnd":"0:00:41.642","start":"0:00:40.574","end":"0:00:41.642"},{"confidence":0.8596,"adjustedStart":"0:00:46.98","adjustedEnd":"0:00:48.048","start":"0:00:46.98","end":"0:00:48.048"},{"confidence":0.8186,"adjustedStart":"0:00:50.183","adjustedEnd":"0:00:51.251","start":"0:00:50.183","end":"0:00:51.251"},{"confidence":0.912,"adjustedStart":"0:00:52.319","adjustedEnd":"0:00:53.387","start":"0:00:52.319","end":"0:00:53.387"},{"confidence":0.8615,"adjustedStart":"0:01:31.825","adjustedEnd":"0:01:33.961","start":"0:01:31.825","end":"0:01:33.961"},{"confidence":0.9449,"adjustedStart":"0:01:39.299","adjustedEnd":"0:01:43.57","start":"0:01:39.299","end":"0:01:43.57"},{"confidence":0.9518,"adjustedStart":"0:02:18.805","adjustedEnd":"0:02:28.415","start":"0:02:18.805","end":"0:02:28.415"},{"confidence":0.9241,"adjustedStart":"0:02:30.55","adjustedEnd":"0:02:31.618","start":"0:02:30.55","end":"0:02:31.618"},{"confidence":0.9154,"adjustedStart":"0:02:39.092","adjustedEnd":"0:02:40.16","start":"0:02:39.092","end":"0:02:40.16"},{"confidence":0.9753,"adjustedStart":"0:02:42.295","adjustedEnd":"0:02:43.363","start":"0:02:42.295","end":"0:02:43.363"},{"confidence":0.893,"adjustedStart":"0:02:52.973","adjustedEnd":"0:02:55.109","start":"0:02:52.973","end":"0:02:55.109"},{"confidence":0.9376,"adjustedStart":"0:02:56.176","adjustedEnd":"0:02:57.244","start":"0:02:56.176","end":"0:02:57.244"},{"confidence":0.9383,"adjustedStart":"0:02:59.379","adjustedEnd":"0:03:02.583","start":"0:02:59.379","end":"0:03:02.583"},{"confidence":0.9158,"adjustedStart":"0:03:06.853","adjustedEnd":"0:03:10.057","start":"0:03:06.853","end":"0:03:10.057"},{"confidence":0.8838,"adjustedStart":"0:03:16.463","adjustedEnd":"0:03:23.937","start":"0:03:16.463","end":"0:03:23.937"},{"confidence":0.8067,"adjustedStart":"0:03:25.005","adjustedEnd":"0:03:26.073","start":"0:03:25.005","end":"0:03:26.073"}]},{"id":50,"name":"carnivore","language":"en-US","instances":[{"confidence":0.9592,"adjustedStart":"0:00:22.422","adjustedEnd":"0:00:30.965","start":"0:00:22.422","end":"0:00:30.965"},{"confidence":0.9386,"adjustedStart":"0:00:33.1","adjustedEnd":"0:00:37.371","start":"0:00:33.1","end":"0:00:37.371"},{"confidence":0.9131,"adjustedStart":"0:00:38.438","adjustedEnd":"0:00:39.506","start":"0:00:38.438","end":"0:00:39.506"},{"confidence":0.9307,"adjustedStart":"0:00:43.777","adjustedEnd":"0:00:44.845","start":"0:00:43.777","end":"0:00:44.845"},{"confidence":0.9453,"adjustedStart":"0:00:45.913","adjustedEnd":"0:00:52.319","start":"0:00:45.913","end":"0:00:52.319"},{"confidence":0.9466,"adjustedStart":"0:01:24.351","adjustedEnd":"0:01:25.419","start":"0:01:24.351","end":"0:01:25.419"},{"confidence":0.9025,"adjustedStart":"0:01:30.757","adjustedEnd":"0:01:31.825","start":"0:01:30.757","end":"0:01:31.825"},{"confidence":0.9235,"adjustedStart":"0:01:32.893","adjustedEnd":"0:01:43.57","start":"0:01:32.893","end":"0:01:43.57"},{"confidence":0.9262,"adjustedStart":"0:02:18.805","adjustedEnd":"0:02:22.009","start":"0:02:18.805","end":"0:02:22.009"},{"confidence":0.916,"adjustedStart":"0:02:24.144","adjustedEnd":"0:02:29.483","start":"0:02:24.144","end":"0:02:29.483"},{"confidence":0.9316,"adjustedStart":"0:02:33.754","adjustedEnd":"0:02:34.822","start":"0:02:33.754","end":"0:02:34.822"},{"confidence":0.906,"adjustedStart":"0:02:39.092","adjustedEnd":"0:02:40.16","start":"0:02:39.092","end":"0:02:40.16"},{"confidence":0.9538,"adjustedStart":"0:02:42.295","adjustedEnd":"0:02:43.363","start":"0:02:42.295","end":"0:02:43.363"},{"confidence":0.9481,"adjustedStart":"0:02:50.837","adjustedEnd":"0:02:51.905","start":"0:02:50.837","end":"0:02:51.905"},{"confidence":0.928,"adjustedStart":"0:02:59.379","adjustedEnd":"0:03:01.515","start":"0:02:59.379","end":"0:03:01.515"},{"confidence":0.911,"adjustedStart":"0:03:03.65","adjustedEnd":"0:03:05.786","start":"0:03:03.65","end":"0:03:05.786"},{"confidence":0.9077,"adjustedStart":"0:03:06.853","adjustedEnd":"0:03:07.921","start":"0:03:06.853","end":"0:03:07.921"},{"confidence":0.9365,"adjustedStart":"0:03:08.989","adjustedEnd":"0:03:13.26","start":"0:03:08.989","end":"0:03:13.26"},{"confidence":0.9295,"adjustedStart":"0:03:14.327","adjustedEnd":"0:03:26.073","start":"0:03:14.327","end":"0:03:26.073"}]},{"id":51,"name":"laying","referenceId":"process/laying","language":"en-US","instances":[{"confidence":0.9584,"adjustedStart":"0:00:25.626","adjustedEnd":"0:00:26.694","start":"0:00:25.626","end":"0:00:26.694"},{"confidence":0.9378,"adjustedStart":"0:01:28.622","adjustedEnd":"0:01:29.69","start":"0:01:28.622","end":"0:01:29.69"},{"confidence":0.9576,"adjustedStart":"0:02:04.925","adjustedEnd":"0:02:09.196","start":"0:02:04.925","end":"0:02:09.196"},{"confidence":0.9401,"adjustedStart":"0:02:51.905","adjustedEnd":"0:02:52.973","start":"0:02:51.905","end":"0:02:52.973"}]},{"id":52,"name":"sleeping","referenceId":"physical condition/sleep/sleeping","language":"en-US","instances":[{"confidence":0.8759,"adjustedStart":"0:00:25.626","adjustedEnd":"0:00:26.694","start":"0:00:25.626","end":"0:00:26.694"}]},{"id":53,"name":"dog breed","language":"en-US","instances":[{"confidence":0.9058,"adjustedStart":"0:00:27.761","adjustedEnd":"0:00:28.829","start":"0:00:27.761","end":"0:00:28.829"},{"confidence":0.9318,"adjustedStart":"0:00:46.98","adjustedEnd":"0:00:48.048","start":"0:00:46.98","end":"0:00:48.048"},{"confidence":0.9013,"adjustedStart":"0:02:19.873","adjustedEnd":"0:02:22.009","start":"0:02:19.873","end":"0:02:22.009"},{"confidence":0.9114,"adjustedStart":"0:02:39.092","adjustedEnd":"0:02:40.16","start":"0:02:39.092","end":"0:02:40.16"},{"confidence":0.918,"adjustedStart":"0:03:23.937","adjustedEnd":"0:03:25.005","start":"0:03:23.937","end":"0:03:25.005"}]},{"id":54,"name":"girl","referenceId":"person/child/girl","language":"en-US","instances":[{"confidence":0.8604,"adjustedStart":"0:00:29.897","adjustedEnd":"0:00:30.965","start":"0:00:29.897","end":"0:00:30.965"},{"confidence":0.8786,"adjustedStart":"0:00:35.235","adjustedEnd":"0:00:36.303","start":"0:00:35.235","end":"0:00:36.303"},{"confidence":0.8552,"adjustedStart":"0:00:42.709","adjustedEnd":"0:00:43.777","start":"0:00:42.709","end":"0:00:43.777"},{"confidence":0.9164,"adjustedStart":"0:01:17.945","adjustedEnd":"0:01:19.013","start":"0:01:17.945","end":"0:01:19.013"},{"confidence":0.8674,"adjustedStart":"0:02:12.399","adjustedEnd":"0:02:13.467","start":"0:02:12.399","end":"0:02:13.467"},{"confidence":0.8724,"adjustedStart":"0:02:35.889","adjustedEnd":"0:02:38.025","start":"0:02:35.889","end":"0:02:38.025"}]},{"id":55,"name":"footwear","referenceId":"consumer goods/clothing/footwear","language":"en-US","instances":[{"confidence":0.9478,"adjustedStart":"0:00:30.964","adjustedEnd":"0:00:32.032","start":"0:00:30.964","end":"0:00:32.032"},{"confidence":0.929,"adjustedStart":"0:00:35.235","adjustedEnd":"0:00:36.303","start":"0:00:35.235","end":"0:00:36.303"}]},{"id":56,"name":"sitting","referenceId":"action/sitting","language":"en-US","instances":[{"confidence":0.8342,"adjustedStart":"0:00:41.642","adjustedEnd":"0:00:42.71","start":"0:00:41.642","end":"0:00:42.71"},{"confidence":0.8942,"adjustedStart":"0:01:33.961","adjustedEnd":"0:01:38.232","start":"0:01:33.961","end":"0:01:38.232"},{"confidence":0.8225,"adjustedStart":"0:02:20.941","adjustedEnd":"0:02:22.009","start":"0:02:20.941","end":"0:02:22.009"}]},{"id":57,"name":"woman","referenceId":"person/woman","language":"en-US","instances":[{"confidence":0.9414,"adjustedStart":"0:00:42.709","adjustedEnd":"0:00:43.777","start":"0:00:42.709","end":"0:00:43.777"},{"confidence":0.8702,"adjustedStart":"0:00:44.845","adjustedEnd":"0:00:46.981","start":"0:00:44.845","end":"0:00:46.981"},{"confidence":0.8838,"adjustedStart":"0:00:48.048","adjustedEnd":"0:00:49.116","start":"0:00:48.048","end":"0:00:49.116"},{"confidence":0.8972,"adjustedStart":"0:01:17.945","adjustedEnd":"0:01:20.08","start":"0:01:17.945","end":"0:01:20.08"},{"confidence":0.8855,"adjustedStart":"0:02:16.67","adjustedEnd":"0:02:17.738","start":"0:02:16.67","end":"0:02:17.738"},{"confidence":0.8974,"adjustedStart":"0:02:34.821","adjustedEnd":"0:02:39.093","start":"0:02:34.821","end":"0:02:39.093"}]},{"id":58,"name":"leash","referenceId":"device/restraint/leash","language":"en-US","instances":[{"confidence":0.9359,"adjustedStart":"0:00:45.913","adjustedEnd":"0:00:48.048","start":"0:00:45.913","end":"0:00:48.048"}]},{"id":59,"name":"terrier","language":"en-US","instances":[{"confidence":0.9103,"adjustedStart":"0:00:46.98","adjustedEnd":"0:00:48.048","start":"0:00:46.98","end":"0:00:48.048"},{"confidence":0.9134,"adjustedStart":"0:01:38.231","adjustedEnd":"0:01:40.367","start":"0:01:38.231","end":"0:01:40.367"},{"confidence":0.9257,"adjustedStart":"0:01:41.435","adjustedEnd":"0:01:43.57","start":"0:01:41.435","end":"0:01:43.57"},{"confidence":0.9284,"adjustedStart":"0:02:18.805","adjustedEnd":"0:02:23.077","start":"0:02:18.805","end":"0:02:23.077"},{"confidence":0.9126,"adjustedStart":"0:02:25.212","adjustedEnd":"0:02:26.28","start":"0:02:25.212","end":"0:02:26.28"},{"confidence":0.9139,"adjustedStart":"0:02:27.347","adjustedEnd":"0:02:29.483","start":"0:02:27.347","end":"0:02:29.483"},{"confidence":0.9294,"adjustedStart":"0:02:39.092","adjustedEnd":"0:02:41.228","start":"0:02:39.092","end":"0:02:41.228"}]},{"id":60,"name":"sporting group","language":"en-US","instances":[{"confidence":0.9032,"adjustedStart":"0:00:46.98","adjustedEnd":"0:00:48.048","start":"0:00:46.98","end":"0:00:48.048"}]},{"id":61,"name":"standing","language":"en-US","instances":[{"confidence":0.8975,"adjustedStart":"0:00:48.048","adjustedEnd":"0:00:49.116","start":"0:00:48.048","end":"0:00:49.116"},{"confidence":0.8795,"adjustedStart":"0:01:48.909","adjustedEnd":"0:01:49.977","start":"0:01:48.909","end":"0:01:49.977"},{"confidence":0.8825,"adjustedStart":"0:03:08.989","adjustedEnd":"0:03:11.125","start":"0:03:08.989","end":"0:03:11.125"},{"confidence":0.8988,"adjustedStart":"0:03:21.802","adjustedEnd":"0:03:22.87","start":"0:03:21.802","end":"0:03:22.87"}]},{"id":62,"name":"indoor","language":"en-US","instances":[{"confidence":0.913,"adjustedStart":"0:00:53.387","adjustedEnd":"0:00:54.455","start":"0:00:53.387","end":"0:00:54.455"},{"confidence":0.9205,"adjustedStart":"0:00:55.522","adjustedEnd":"0:00:56.59","start":"0:00:55.522","end":"0:00:56.59"},{"confidence":0.9482,"adjustedStart":"0:00:59.793","adjustedEnd":"0:01:00.861","start":"0:00:59.793","end":"0:01:00.861"},{"confidence":0.8395,"adjustedStart":"0:01:01.929","adjustedEnd":"0:01:02.997","start":"0:01:01.929","end":"0:01:02.997"},{"confidence":0.8508,"adjustedStart":"0:01:21.148","adjustedEnd":"0:01:22.216","start":"0:01:21.148","end":"0:01:22.216"},{"confidence":0.8643,"adjustedStart":"0:01:23.283","adjustedEnd":"0:01:24.351","start":"0:01:23.283","end":"0:01:24.351"},{"confidence":0.8959,"adjustedStart":"0:01:38.231","adjustedEnd":"0:01:39.299","start":"0:01:38.231","end":"0:01:39.299"},{"confidence":0.9109,"adjustedStart":"0:01:40.367","adjustedEnd":"0:01:52.112","start":"0:01:40.367","end":"0:01:52.112"},{"confidence":0.8171,"adjustedStart":"0:01:53.18","adjustedEnd":"0:01:54.248","start":"0:01:53.18","end":"0:01:54.248"},{"confidence":0.8945,"adjustedStart":"0:01:58.518","adjustedEnd":"0:02:09.196","start":"0:01:58.518","end":"0:02:09.196"},{"confidence":0.8891,"adjustedStart":"0:02:13.467","adjustedEnd":"0:02:24.144","start":"0:02:13.467","end":"0:02:24.144"},{"confidence":0.8379,"adjustedStart":"0:02:25.212","adjustedEnd":"0:02:28.415","start":"0:02:25.212","end":"0:02:28.415"},{"confidence":0.908,"adjustedStart":"0:02:29.483","adjustedEnd":"0:02:30.551","start":"0:02:29.483","end":"0:02:30.551"},{"confidence":0.9347,"adjustedStart":"0:02:32.686","adjustedEnd":"0:02:39.093","start":"0:02:32.686","end":"0:02:39.093"},{"confidence":0.8958,"adjustedStart":"0:02:47.634","adjustedEnd":"0:02:55.109","start":"0:02:47.634","end":"0:02:55.109"},{"confidence":0.9008,"adjustedStart":"0:02:56.176","adjustedEnd":"0:02:57.244","start":"0:02:56.176","end":"0:02:57.244"},{"confidence":0.8865,"adjustedStart":"0:02:58.311","adjustedEnd":"0:03:02.583","start":"0:02:58.311","end":"0:03:02.583"},{"confidence":0.9478,"adjustedStart":"0:03:04.718","adjustedEnd":"0:03:16.463","start":"0:03:04.718","end":"0:03:16.463"},{"confidence":0.9855,"adjustedStart":"0:03:19.666","adjustedEnd":"0:03:27.141","start":"0:03:19.666","end":"0:03:27.141"}]},{"id":63,"name":"jacket","referenceId":"clothing/coat/jacket","language":"en-US","instances":[{"confidence":0.9063,"adjustedStart":"0:01:08.335","adjustedEnd":"0:01:09.403","start":"0:01:08.335","end":"0:01:09.403"}]},{"id":64,"name":"pc game","language":"en-US","instances":[{"confidence":0.9189,"adjustedStart":"0:01:12.606","adjustedEnd":"0:01:13.674","start":"0:01:12.606","end":"0:01:13.674"},{"confidence":0.9256,"adjustedStart":"0:01:26.486","adjustedEnd":"0:01:27.554","start":"0:01:26.486","end":"0:01:27.554"}]},{"id":65,"name":"cartoon","language":"en-US","instances":[{"confidence":0.9045,"adjustedStart":"0:01:12.606","adjustedEnd":"0:01:13.674","start":"0:01:12.606","end":"0:01:13.674"},{"confidence":0.916,"adjustedStart":"0:01:26.486","adjustedEnd":"0:01:27.554","start":"0:01:26.486","end":"0:01:27.554"}]},{"id":66,"name":"human face","language":"en-US","instances":[{"confidence":0.9682,"adjustedStart":"0:01:13.674","adjustedEnd":"0:01:14.742","start":"0:01:13.674","end":"0:01:14.742"}]},{"id":67,"name":"smile","language":"en-US","instances":[{"confidence":0.907,"adjustedStart":"0:01:13.674","adjustedEnd":"0:01:14.742","start":"0:01:13.674","end":"0:01:14.742"}]},{"id":68,"name":"accessory","referenceId":"covering/clothing/accessory","language":"en-US","instances":[{"confidence":0.8457,"adjustedStart":"0:01:13.674","adjustedEnd":"0:01:14.742","start":"0:01:13.674","end":"0:01:14.742"}]},{"id":69,"name":"fashion accessory","language":"en-US","instances":[{"confidence":0.9047,"adjustedStart":"0:01:19.012","adjustedEnd":"0:01:20.08","start":"0:01:19.012","end":"0:01:20.08"}]},{"id":70,"name":"window","referenceId":"structure/framework/window","language":"en-US","instances":[{"confidence":0.9308,"adjustedStart":"0:01:21.148","adjustedEnd":"0:01:22.216","start":"0:01:21.148","end":"0:01:22.216"},{"confidence":0.938,"adjustedStart":"0:01:32.893","adjustedEnd":"0:01:33.961","start":"0:01:32.893","end":"0:01:33.961"}]},{"id":71,"name":"shadow","language":"en-US","instances":[{"confidence":0.997,"adjustedStart":"0:01:25.419","adjustedEnd":"0:01:26.487","start":"0:01:25.419","end":"0:01:26.487"}]},{"id":72,"name":"seat","referenceId":"location/region/area/space/seat","language":"en-US","instances":[{"confidence":0.9155,"adjustedStart":"0:01:30.757","adjustedEnd":"0:01:31.825","start":"0:01:30.757","end":"0:01:31.825"},{"confidence":0.9719,"adjustedStart":"0:01:32.893","adjustedEnd":"0:01:38.232","start":"0:01:32.893","end":"0:01:38.232"}]},{"id":73,"name":"hair","referenceId":"body covering/hair","language":"en-US","instances":[{"confidence":0.9097,"adjustedStart":"0:01:39.299","adjustedEnd":"0:01:43.57","start":"0:01:39.299","end":"0:01:43.57"},{"confidence":0.9016,"adjustedStart":"0:02:12.399","adjustedEnd":"0:02:13.467","start":"0:02:12.399","end":"0:02:13.467"},{"confidence":0.8797,"adjustedStart":"0:02:23.076","adjustedEnd":"0:02:24.144","start":"0:02:23.076","end":"0:02:24.144"}]},{"id":74,"name":"puppy","language":"en-US","instances":[{"confidence":0.9861,"adjustedStart":"0:01:40.367","adjustedEnd":"0:01:41.435","start":"0:01:40.367","end":"0:01:41.435"},{"confidence":0.9928,"adjustedStart":"0:02:18.805","adjustedEnd":"0:02:22.009","start":"0:02:18.805","end":"0:02:22.009"},{"confidence":0.9859,"adjustedStart":"0:02:26.279","adjustedEnd":"0:02:27.347","start":"0:02:26.279","end":"0:02:27.347"},{"confidence":0.9913,"adjustedStart":"0:03:08.989","adjustedEnd":"0:03:13.26","start":"0:03:08.989","end":"0:03:13.26"},{"confidence":0.9905,"adjustedStart":"0:03:20.734","adjustedEnd":"0:03:26.073","start":"0:03:20.734","end":"0:03:26.073"}]},{"id":75,"name":"havanese","language":"en-US","instances":[{"confidence":0.9014,"adjustedStart":"0:01:42.502","adjustedEnd":"0:01:43.57","start":"0:01:42.502","end":"0:01:43.57"},{"confidence":0.9132,"adjustedStart":"0:02:18.805","adjustedEnd":"0:02:20.941","start":"0:02:18.805","end":"0:02:20.941"}]},{"id":76,"name":"home appliance","language":"en-US","instances":[{"confidence":0.9407,"adjustedStart":"0:01:44.638","adjustedEnd":"0:01:45.706","start":"0:01:44.638","end":"0:01:45.706"}]},{"id":77,"name":"sink","referenceId":"sink","language":"en-US","instances":[{"confidence":0.9333,"adjustedStart":"0:01:44.638","adjustedEnd":"0:01:45.706","start":"0:01:44.638","end":"0:01:45.706"}]},{"id":78,"name":"kitchen appliance","referenceId":"consumer goods/appliance/kitchen appliance/kitchen appliance","language":"en-US","instances":[{"confidence":0.9273,"adjustedStart":"0:01:44.638","adjustedEnd":"0:01:45.706","start":"0:01:44.638","end":"0:01:45.706"}]},{"id":79,"name":"blue","referenceId":"color/blue","language":"en-US","instances":[{"confidence":0.939,"adjustedStart":"0:01:48.909","adjustedEnd":"0:01:49.977","start":"0:01:48.909","end":"0:01:49.977"}]},{"id":80,"name":"laptop","referenceId":"computer/portable computer/laptop","language":"en-US","instances":[{"confidence":0.9907,"adjustedStart":"0:01:51.044","adjustedEnd":"0:01:54.248","start":"0:01:51.044","end":"0:01:54.248"}]},{"id":81,"name":"television","referenceId":"television","language":"en-US","instances":[{"confidence":0.9894,"adjustedStart":"0:01:51.044","adjustedEnd":"0:01:54.248","start":"0:01:51.044","end":"0:01:54.248"},{"confidence":0.8086,"adjustedStart":"0:03:20.734","adjustedEnd":"0:03:21.802","start":"0:03:20.734","end":"0:03:21.802"},{"confidence":0.8475,"adjustedStart":"0:03:26.073","adjustedEnd":"0:03:27.141","start":"0:03:26.073","end":"0:03:27.141"}]},{"id":82,"name":"electronics","language":"en-US","instances":[{"confidence":0.961,"adjustedStart":"0:01:51.044","adjustedEnd":"0:01:54.248","start":"0:01:51.044","end":"0:01:54.248"},{"confidence":0.9239,"adjustedStart":"0:01:59.586","adjustedEnd":"0:02:00.654","start":"0:01:59.586","end":"0:02:00.654"}]},{"id":83,"name":"monitor","referenceId":"monitor","language":"en-US","instances":[{"confidence":0.9217,"adjustedStart":"0:01:51.044","adjustedEnd":"0:01:53.18","start":"0:01:51.044","end":"0:01:53.18"},{"confidence":0.9457,"adjustedStart":"0:01:59.586","adjustedEnd":"0:02:00.654","start":"0:01:59.586","end":"0:02:00.654"}]},{"id":84,"name":"display","referenceId":"communication/demonstration/display","language":"en-US","instances":[{"confidence":0.9194,"adjustedStart":"0:01:51.044","adjustedEnd":"0:01:52.112","start":"0:01:51.044","end":"0:01:52.112"},{"confidence":0.9111,"adjustedStart":"0:01:53.18","adjustedEnd":"0:01:55.315","start":"0:01:53.18","end":"0:01:55.315"},{"confidence":0.9582,"adjustedStart":"0:01:58.518","adjustedEnd":"0:02:00.654","start":"0:01:58.518","end":"0:02:00.654"}]},{"id":85,"name":"display device","language":"en-US","instances":[{"confidence":0.9215,"adjustedStart":"0:01:51.044","adjustedEnd":"0:01:54.248","start":"0:01:51.044","end":"0:01:54.248"},{"confidence":0.9008,"adjustedStart":"0:01:59.586","adjustedEnd":"0:02:00.654","start":"0:01:59.586","end":"0:02:00.654"}]},{"id":86,"name":"computer","referenceId":"device/machine/computer","language":"en-US","instances":[{"confidence":0.8615,"adjustedStart":"0:01:51.044","adjustedEnd":"0:01:54.248","start":"0:01:51.044","end":"0:01:54.248"},{"confidence":0.9086,"adjustedStart":"0:01:58.518","adjustedEnd":"0:02:00.654","start":"0:01:58.518","end":"0:02:00.654"}]},{"id":87,"name":"screen","referenceId":"electronics/screen","language":"en-US","instances":[{"confidence":0.9891,"adjustedStart":"0:01:52.112","adjustedEnd":"0:01:53.18","start":"0:01:52.112","end":"0:01:53.18"},{"confidence":0.9687,"adjustedStart":"0:01:58.518","adjustedEnd":"0:02:00.654","start":"0:01:58.518","end":"0:02:00.654"}]},{"id":88,"name":"mac","language":"en-US","instances":[{"confidence":0.9884,"adjustedStart":"0:01:52.112","adjustedEnd":"0:01:54.248","start":"0:01:52.112","end":"0:01:54.248"}]},{"id":89,"name":"multimedia","language":"en-US","instances":[{"confidence":0.9308,"adjustedStart":"0:01:52.112","adjustedEnd":"0:01:54.248","start":"0:01:52.112","end":"0:01:54.248"}]},{"id":90,"name":"design","language":"en-US","instances":[{"confidence":0.9968,"adjustedStart":"0:01:54.247","adjustedEnd":"0:01:55.315","start":"0:01:54.247","end":"0:01:55.315"},{"confidence":0.9877,"adjustedStart":"0:01:56.383","adjustedEnd":"0:01:57.451","start":"0:01:56.383","end":"0:01:57.451"},{"confidence":0.9873,"adjustedStart":"0:03:30.343","adjustedEnd":"0:03:31.411","start":"0:03:30.343","end":"0:03:31.411"}]},{"id":91,"name":"man","referenceId":"person/man","language":"en-US","instances":[{"confidence":0.9421,"adjustedStart":"0:01:56.383","adjustedEnd":"0:01:57.451","start":"0:01:56.383","end":"0:01:57.451"},{"confidence":0.8581,"adjustedStart":"0:02:48.702","adjustedEnd":"0:02:49.77","start":"0:02:48.702","end":"0:02:49.77"},{"confidence":0.8885,"adjustedStart":"0:03:02.582","adjustedEnd":"0:03:03.65","start":"0:03:02.582","end":"0:03:03.65"}]},{"id":92,"name":"output device","language":"en-US","instances":[{"confidence":0.9246,"adjustedStart":"0:01:59.586","adjustedEnd":"0:02:00.654","start":"0:01:59.586","end":"0:02:00.654"}]},{"id":93,"name":"medical equipment","language":"en-US","instances":[{"confidence":0.9262,"adjustedStart":"0:02:00.654","adjustedEnd":"0:02:01.722","start":"0:02:00.654","end":"0:02:01.722"},{"confidence":0.9454,"adjustedStart":"0:02:02.789","adjustedEnd":"0:02:04.925","start":"0:02:02.789","end":"0:02:04.925"},{"confidence":0.9407,"adjustedStart":"0:02:15.602","adjustedEnd":"0:02:17.738","start":"0:02:15.602","end":"0:02:17.738"}]},{"id":94,"name":"medical","language":"en-US","instances":[{"confidence":0.9316,"adjustedStart":"0:02:02.789","adjustedEnd":"0:02:04.925","start":"0:02:02.789","end":"0:02:04.925"},{"confidence":0.9183,"adjustedStart":"0:02:15.602","adjustedEnd":"0:02:16.67","start":"0:02:15.602","end":"0:02:16.67"}]},{"id":95,"name":"health care","language":"en-US","instances":[{"confidence":0.9063,"adjustedStart":"0:02:02.789","adjustedEnd":"0:02:04.925","start":"0:02:02.789","end":"0:02:04.925"}]},{"id":96,"name":"bedclothes","referenceId":"covering/cloth covering/bedclothes","language":"en-US","instances":[{"confidence":0.943,"adjustedStart":"0:02:04.925","adjustedEnd":"0:02:09.196","start":"0:02:04.925","end":"0:02:09.196"}]},{"id":97,"name":"bed","referenceId":"furniture/bed","language":"en-US","instances":[{"confidence":0.8825,"adjustedStart":"0:02:04.925","adjustedEnd":"0:02:09.196","start":"0:02:04.925","end":"0:02:09.196"}]},{"id":98,"name":"nude","language":"en-US","instances":[{"confidence":0.9889,"adjustedStart":"0:02:05.993","adjustedEnd":"0:02:07.061","start":"0:02:05.993","end":"0:02:07.061"}]},{"id":99,"name":"medical procedure","language":"en-US","instances":[{"confidence":0.938,"adjustedStart":"0:02:15.602","adjustedEnd":"0:02:16.67","start":"0:02:15.602","end":"0:02:16.67"}]},{"id":100,"name":"medical glove","language":"en-US","instances":[{"confidence":0.9294,"adjustedStart":"0:02:15.602","adjustedEnd":"0:02:16.67","start":"0:02:15.602","end":"0:02:16.67"}]},{"id":101,"name":"patient","language":"en-US","instances":[{"confidence":0.913,"adjustedStart":"0:02:15.602","adjustedEnd":"0:02:16.67","start":"0:02:15.602","end":"0:02:16.67"}]},{"id":102,"name":"surgery","language":"en-US","instances":[{"confidence":0.901,"adjustedStart":"0:02:15.602","adjustedEnd":"0:02:16.67","start":"0:02:15.602","end":"0:02:16.67"}]},{"id":103,"name":"shih-poo","language":"en-US","instances":[{"confidence":0.9193,"adjustedStart":"0:02:18.805","adjustedEnd":"0:02:20.941","start":"0:02:18.805","end":"0:02:20.941"}]},{"id":104,"name":"west highland white terrier","language":"en-US","instances":[{"confidence":0.9129,"adjustedStart":"0:02:18.805","adjustedEnd":"0:02:22.009","start":"0:02:18.805","end":"0:02:22.009"},{"confidence":0.9033,"adjustedStart":"0:02:28.415","adjustedEnd":"0:02:29.483","start":"0:02:28.415","end":"0:02:29.483"}]},{"id":105,"name":"lhasa apso","language":"en-US","instances":[{"confidence":0.9098,"adjustedStart":"0:02:18.805","adjustedEnd":"0:02:20.941","start":"0:02:18.805","end":"0:02:20.941"}]},{"id":106,"name":"sporting lucas terrier","language":"en-US","instances":[{"confidence":0.9084,"adjustedStart":"0:02:18.805","adjustedEnd":"0:02:20.941","start":"0:02:18.805","end":"0:02:20.941"}]},{"id":107,"name":"toy dog","language":"en-US","instances":[{"confidence":0.9099,"adjustedStart":"0:02:18.805","adjustedEnd":"0:02:20.941","start":"0:02:18.805","end":"0:02:20.941"},{"confidence":0.904,"adjustedStart":"0:02:39.092","adjustedEnd":"0:02:40.16","start":"0:02:39.092","end":"0:02:40.16"}]},{"id":108,"name":"yorkipoo","language":"en-US","instances":[{"confidence":0.9064,"adjustedStart":"0:02:18.805","adjustedEnd":"0:02:19.873","start":"0:02:18.805","end":"0:02:19.873"}]},{"id":109,"name":"tibetan terrier","language":"en-US","instances":[{"confidence":0.9068,"adjustedStart":"0:02:18.805","adjustedEnd":"0:02:20.941","start":"0:02:18.805","end":"0:02:20.941"}]},{"id":110,"name":"schnoodle","language":"en-US","instances":[{"confidence":0.9029,"adjustedStart":"0:02:18.805","adjustedEnd":"0:02:20.941","start":"0:02:18.805","end":"0:02:20.941"}]},{"id":111,"name":"poodle crossbreed","language":"en-US","instances":[{"confidence":0.9016,"adjustedStart":"0:02:18.805","adjustedEnd":"0:02:20.941","start":"0:02:18.805","end":"0:02:20.941"}]},{"id":112,"name":"coton de tulear","language":"en-US","instances":[{"confidence":0.9045,"adjustedStart":"0:02:18.805","adjustedEnd":"0:02:20.941","start":"0:02:18.805","end":"0:02:20.941"}]},{"id":113,"name":"maltese","language":"en-US","instances":[{"confidence":0.9035,"adjustedStart":"0:02:19.873","adjustedEnd":"0:02:20.941","start":"0:02:19.873","end":"0:02:20.941"}]},{"id":114,"name":"lawchen","language":"en-US","instances":[{"confidence":0.9016,"adjustedStart":"0:02:19.873","adjustedEnd":"0:02:20.941","start":"0:02:19.873","end":"0:02:20.941"}]},{"id":115,"name":"kyi-leo","language":"en-US","instances":[{"confidence":0.9007,"adjustedStart":"0:02:19.873","adjustedEnd":"0:02:20.941","start":"0:02:19.873","end":"0:02:20.941"}]},{"id":116,"name":"wearing","language":"en-US","instances":[{"confidence":0.9238,"adjustedStart":"0:02:27.347","adjustedEnd":"0:02:28.415","start":"0:02:27.347","end":"0:02:28.415"}]},{"id":117,"name":"white","referenceId":"white","language":"en-US","instances":[{"confidence":0.9277,"adjustedStart":"0:02:28.415","adjustedEnd":"0:02:29.483","start":"0:02:28.415","end":"0:02:29.483"}]},{"id":118,"name":"legs","referenceId":"body parts/legs","language":"en-US","instances":[{"confidence":0.9187,"adjustedStart":"0:02:29.483","adjustedEnd":"0:02:30.551","start":"0:02:29.483","end":"0:02:30.551"}]},{"id":119,"name":"green","referenceId":"color/green","language":"en-US","instances":[{"confidence":0.9816,"adjustedStart":"0:02:38.025","adjustedEnd":"0:02:39.093","start":"0:02:38.025","end":"0:02:39.093"},{"confidence":0.9217,"adjustedStart":"0:02:51.905","adjustedEnd":"0:02:52.973","start":"0:02:51.905","end":"0:02:52.973"},{"confidence":0.9462,"adjustedStart":"0:02:54.041","adjustedEnd":"0:02:55.109","start":"0:02:54.041","end":"0:02:55.109"}]},{"id":120,"name":"dance","language":"en-US","instances":[{"confidence":0.9458,"adjustedStart":"0:02:38.025","adjustedEnd":"0:02:39.093","start":"0:02:38.025","end":"0:02:39.093"}]},{"id":121,"name":"vulnerable native breeds","language":"en-US","instances":[{"confidence":0.9113,"adjustedStart":"0:02:39.092","adjustedEnd":"0:02:40.16","start":"0:02:39.092","end":"0:02:40.16"}]},{"id":122,"name":"pet","language":"en-US","instances":[{"confidence":0.9088,"adjustedStart":"0:02:39.092","adjustedEnd":"0:02:40.16","start":"0:02:39.092","end":"0:02:40.16"}]},{"id":123,"name":"blurry","language":"en-US","instances":[{"confidence":0.9135,"adjustedStart":"0:02:44.431","adjustedEnd":"0:02:45.499","start":"0:02:44.431","end":"0:02:45.499"}]},{"id":124,"name":"little","referenceId":"measure/little","language":"en-US","instances":[{"confidence":0.9409,"adjustedStart":"0:02:58.311","adjustedEnd":"0:02:59.379","start":"0:02:58.311","end":"0:02:59.379"}]},{"id":125,"name":"remote","referenceId":"instrumentality/device/remote control/remote","language":"en-US","instances":[{"confidence":0.8882,"adjustedStart":"0:03:14.327","adjustedEnd":"0:03:15.395","start":"0:03:14.327","end":"0:03:15.395"}]},{"id":126,"name":"room","referenceId":"structure/room","language":"en-US","instances":[{"confidence":0.951,"adjustedStart":"0:03:20.734","adjustedEnd":"0:03:21.802","start":"0:03:20.734","end":"0:03:21.802"},{"confidence":0.9298,"adjustedStart":"0:03:23.937","adjustedEnd":"0:03:25.005","start":"0:03:23.937","end":"0:03:25.005"}]},{"id":127,"name":"shelf","referenceId":"furniture/shelf","language":"en-US","instances":[{"confidence":0.8265,"adjustedStart":"0:03:25.005","adjustedEnd":"0:03:27.141","start":"0:03:25.005","end":"0:03:27.141"}]},{"id":128,"name":"space","referenceId":"space","language":"en-US","instances":[{"confidence":0.9345,"adjustedStart":"0:03:27.14","adjustedEnd":"0:03:28.208","start":"0:03:27.14","end":"0:03:28.208"}]},{"id":129,"name":"fractal art","language":"en-US","instances":[{"confidence":0.9104,"adjustedStart":"0:03:27.14","adjustedEnd":"0:03:28.208","start":"0:03:27.14","end":"0:03:28.208"}]},{"id":130,"name":"bright","language":"en-US","instances":[{"confidence":0.8467,"adjustedStart":"0:03:27.14","adjustedEnd":"0:03:28.208","start":"0:03:27.14","end":"0:03:28.208"}]},{"id":131,"name":"compact disk","referenceId":"memory device/optical disk/compact disk/compact disk","language":"en-US","instances":[{"confidence":0.9529,"adjustedStart":"0:03:28.208","adjustedEnd":"0:03:29.276","start":"0:03:28.208","end":"0:03:29.276"}]}],"scenes":[{"id":1,"instances":[{"adjustedStart":"0:00:00","adjustedEnd":"0:00:05.672","start":"0:00:00","end":"0:00:05.672"}]},{"id":2,"instances":[{"adjustedStart":"0:00:05.672","adjustedEnd":"0:00:09.776","start":"0:00:05.672","end":"0:00:09.776"}]},{"id":3,"instances":[{"adjustedStart":"0:00:09.776","adjustedEnd":"0:01:03.53","start":"0:00:09.776","end":"0:01:03.53"}]},{"id":4,"instances":[{"adjustedStart":"0:01:03.53","adjustedEnd":"0:01:25.185","start":"0:01:03.53","end":"0:01:25.185"}]},{"id":5,"instances":[{"adjustedStart":"0:01:25.185","adjustedEnd":"0:01:37.798","start":"0:01:25.185","end":"0:01:37.798"}]},{"id":6,"instances":[{"adjustedStart":"0:01:37.798","adjustedEnd":"0:02:00.454","start":"0:01:37.798","end":"0:02:00.454"}]},{"id":7,"instances":[{"adjustedStart":"0:02:00.454","adjustedEnd":"0:02:27.614","start":"0:02:00.454","end":"0:02:27.614"}]},{"id":8,"instances":[{"adjustedStart":"0:02:27.614","adjustedEnd":"0:02:58.912","start":"0:02:27.614","end":"0:02:58.912"}]},{"id":9,"instances":[{"adjustedStart":"0:02:58.912","adjustedEnd":"0:03:26.406","start":"0:02:58.912","end":"0:03:26.406"}]},{"id":10,"instances":[{"adjustedStart":"0:03:26.406","adjustedEnd":"0:03:31.978","start":"0:03:26.406","end":"0:03:31.978"}]}],"shots":[{"id":1,"keyFrames":[{"id":1,"instances":[{"thumbnailId":"536192a2-40c3-44c6-8ecd-7576fd99ebc5","adjustedStart":"0:00:00.434","adjustedEnd":"0:00:00.467","start":"0:00:00.434","end":"0:00:00.467"}]},{"id":2,"instances":[{"thumbnailId":"300eb49c-19e2-4041-82b9-4fd97ee5b6fb","adjustedStart":"0:00:00.667","adjustedEnd":"0:00:00.7","start":"0:00:00.667","end":"0:00:00.7"}]},{"id":3,"instances":[{"thumbnailId":"53be4958-f2ca-43e8-8ad7-1c6fcfd928d7","adjustedStart":"0:00:01.168","adjustedEnd":"0:00:01.201","start":"0:00:01.168","end":"0:00:01.201"}]}],"instances":[{"adjustedStart":"0:00:00","adjustedEnd":"0:00:02.035","start":"0:00:00","end":"0:00:02.035"}]},{"id":2,"keyFrames":[{"id":4,"instances":[{"thumbnailId":"8f24c1c3-b4b0-49d8-8e9a-f994aa59c836","adjustedStart":"0:00:02.035","adjustedEnd":"0:00:02.068","start":"0:00:02.035","end":"0:00:02.068"}]},{"id":5,"instances":[{"thumbnailId":"8cc676c7-5983-42ee-b8bf-c1bec389704d","adjustedStart":"0:00:03.203","adjustedEnd":"0:00:03.236","start":"0:00:03.203","end":"0:00:03.236"}]}],"instances":[{"adjustedStart":"0:00:02.035","adjustedEnd":"0:00:05.672","start":"0:00:02.035","end":"0:00:05.672"}]},{"id":3,"tags":["Outdoor"],"keyFrames":[{"id":6,"instances":[{"thumbnailId":"6611554a-f9f6-4ccc-a078-bf8077ddc5c2","adjustedStart":"0:00:05.672","adjustedEnd":"0:00:05.705","start":"0:00:05.672","end":"0:00:05.705"}]},{"id":7,"instances":[{"thumbnailId":"5549dc98-5cd0-4064-a0ef-b29cb2a1dacb","adjustedStart":"0:00:08.175","adjustedEnd":"0:00:08.208","start":"0:00:08.175","end":"0:00:08.208"}]},{"id":8,"instances":[{"thumbnailId":"60356427-53ac-4991-a27f-7fcec80d266e","adjustedStart":"0:00:09.543","adjustedEnd":"0:00:09.576","start":"0:00:09.543","end":"0:00:09.576"}]}],"instances":[{"adjustedStart":"0:00:05.672","adjustedEnd":"0:00:09.776","start":"0:00:05.672","end":"0:00:09.776"}]},{"id":4,"tags":["Outdoor"],"keyFrames":[{"id":9,"instances":[{"thumbnailId":"7338c915-42cf-4be8-b083-89ad228ff8da","adjustedStart":"0:00:09.776","adjustedEnd":"0:00:09.809","start":"0:00:09.776","end":"0:00:09.809"}]},{"id":10,"instances":[{"thumbnailId":"f4334fca-ceff-48e4-ba3f-34799e35a285","adjustedStart":"0:00:12.846","adjustedEnd":"0:00:12.879","start":"0:00:12.846","end":"0:00:12.879"}]},{"id":11,"instances":[{"thumbnailId":"c7f34bf3-4012-4c22-a25d-8f941802ba7f","adjustedStart":"0:00:15.115","adjustedEnd":"0:00:15.148","start":"0:00:15.115","end":"0:00:15.148"}]}],"instances":[{"adjustedStart":"0:00:09.776","adjustedEnd":"0:00:18.285","start":"0:00:09.776","end":"0:00:18.285"}]},{"id":5,"tags":["Outdoor"],"keyFrames":[{"id":12,"instances":[{"thumbnailId":"db9f1a9e-b258-461e-b6d3-5f0b262439a3","adjustedStart":"0:00:18.285","adjustedEnd":"0:00:18.318","start":"0:00:18.285","end":"0:00:18.318"}]},{"id":13,"instances":[{"thumbnailId":"492f5142-3d1e-4430-9471-acaf0c27ea83","adjustedStart":"0:00:26.426","adjustedEnd":"0:00:26.459","start":"0:00:26.426","end":"0:00:26.459"}]},{"id":14,"instances":[{"thumbnailId":"0f14657e-7429-4492-8563-4df06cd8b8cc","adjustedStart":"0:00:35.869","adjustedEnd":"0:00:35.902","start":"0:00:35.869","end":"0:00:35.902"}]}],"instances":[{"adjustedStart":"0:00:18.285","adjustedEnd":"0:00:38.205","start":"0:00:18.285","end":"0:00:38.205"}]},{"id":6,"tags":["Indoor","Outdoor"],"keyFrames":[{"id":15,"instances":[{"thumbnailId":"790f3cb0-5cb1-469a-af84-c2b2ee48b4a8","adjustedStart":"0:00:38.205","adjustedEnd":"0:00:38.238","start":"0:00:38.205","end":"0:00:38.238"}]},{"id":16,"instances":[{"thumbnailId":"17d435ef-6d0d-4d28-b502-aabc0298e5ce","adjustedStart":"0:00:42.643","adjustedEnd":"0:00:42.676","start":"0:00:42.643","end":"0:00:42.676"}]},{"id":17,"instances":[{"thumbnailId":"2d661905-b532-4898-b917-ea9ef1d9e9de","adjustedStart":"0:00:48.749","adjustedEnd":"0:00:48.782","start":"0:00:48.749","end":"0:00:48.782"}]}],"instances":[{"adjustedStart":"0:00:38.205","adjustedEnd":"0:00:52.219","start":"0:00:38.205","end":"0:00:52.219"}]},{"id":7,"tags":["Indoor"],"keyFrames":[{"id":18,"instances":[{"thumbnailId":"9deec3c9-f883-4f17-befd-05e9e680cc2e","adjustedStart":"0:00:52.219","adjustedEnd":"0:00:52.252","start":"0:00:52.219","end":"0:00:52.252"}]},{"id":19,"instances":[{"thumbnailId":"6824208b-22ff-4a0c-a4cc-4518b128b5f7","adjustedStart":"0:00:56.189","adjustedEnd":"0:00:56.222","start":"0:00:56.189","end":"0:00:56.222"}]},{"id":20,"instances":[{"thumbnailId":"9f246452-f0a5-4396-b0d3-a4630948d2ba","adjustedStart":"0:01:02.663","adjustedEnd":"0:01:02.696","start":"0:01:02.663","end":"0:01:02.696"}]}],"instances":[{"adjustedStart":"0:00:52.219","adjustedEnd":"0:01:03.53","start":"0:00:52.219","end":"0:01:03.53"}]},{"id":8,"keyFrames":[{"id":21,"instances":[{"thumbnailId":"8266b2d4-a224-44a5-bb3d-e94c46d125f4","adjustedStart":"0:01:03.53","adjustedEnd":"0:01:03.563","start":"0:01:03.53","end":"0:01:03.563"}]},{"id":22,"instances":[{"thumbnailId":"31524080-e1c9-41bb-b3ad-47999ed662a2","adjustedStart":"0:01:08.201","adjustedEnd":"0:01:08.234","start":"0:01:08.201","end":"0:01:08.234"}]},{"id":23,"instances":[{"thumbnailId":"602392f9-a38a-415b-9625-0e91c0827ff7","adjustedStart":"0:01:11.405","adjustedEnd":"0:01:11.438","start":"0:01:11.405","end":"0:01:11.438"}]}],"instances":[{"adjustedStart":"0:01:03.53","adjustedEnd":"0:01:13.14","start":"0:01:03.53","end":"0:01:13.14"}]},{"id":9,"tags":["Outdoor"],"keyFrames":[{"id":24,"instances":[{"thumbnailId":"0cf79176-60b8-4e3a-8786-5543b9459179","adjustedStart":"0:01:13.14","adjustedEnd":"0:01:13.173","start":"0:01:13.14","end":"0:01:13.173"}]},{"id":25,"instances":[{"thumbnailId":"43e94c65-2203-4841-bd5c-8dd479f56785","adjustedStart":"0:01:14.274","adjustedEnd":"0:01:14.307","start":"0:01:14.274","end":"0:01:14.307"}]},{"id":26,"instances":[{"thumbnailId":"75667ee4-c120-433a-b17b-2f60716d1976","adjustedStart":"0:01:15.542","adjustedEnd":"0:01:15.575","start":"0:01:15.542","end":"0:01:15.575"}]}],"instances":[{"adjustedStart":"0:01:13.14","adjustedEnd":"0:01:16.61","start":"0:01:13.14","end":"0:01:16.61"}]},{"id":10,"keyFrames":[{"id":27,"instances":[{"thumbnailId":"fad2e477-498c-4b67-9c99-2393ad96c9b6","adjustedStart":"0:01:16.61","adjustedEnd":"0:01:16.643","start":"0:01:16.61","end":"0:01:16.643"}]},{"id":28,"instances":[{"thumbnailId":"adf5157b-f665-4985-b5f3-29af87e09c4d","adjustedStart":"0:01:18.679","adjustedEnd":"0:01:18.712","start":"0:01:18.679","end":"0:01:18.712"}]},{"id":29,"instances":[{"thumbnailId":"42847596-7088-4cdd-954a-367ee397268d","adjustedStart":"0:01:20.714","adjustedEnd":"0:01:20.747","start":"0:01:20.714","end":"0:01:20.747"}]}],"instances":[{"adjustedStart":"0:01:16.61","adjustedEnd":"0:01:21.248","start":"0:01:16.61","end":"0:01:21.248"}]},{"id":11,"tags":["Indoor"],"keyFrames":[{"id":30,"instances":[{"thumbnailId":"1185df39-7565-45e6-8992-5f7b769e9121","adjustedStart":"0:01:21.248","adjustedEnd":"0:01:21.281","start":"0:01:21.248","end":"0:01:21.281"}]},{"id":31,"instances":[{"thumbnailId":"df2c3c8e-3c53-43ae-be2c-efb87e6577fe","adjustedStart":"0:01:23.383","adjustedEnd":"0:01:23.416","start":"0:01:23.383","end":"0:01:23.416"}]},{"id":32,"instances":[{"thumbnailId":"4e55b4f0-0079-48bd-a5b2-a456aed65798","adjustedStart":"0:01:24.651","adjustedEnd":"0:01:24.684","start":"0:01:24.651","end":"0:01:24.684"}]}],"instances":[{"adjustedStart":"0:01:21.248","adjustedEnd":"0:01:25.185","start":"0:01:21.248","end":"0:01:25.185"}]},{"id":12,"keyFrames":[{"id":33,"instances":[{"thumbnailId":"7ad88f99-b016-4fa7-aee7-0888bd9350ae","adjustedStart":"0:01:25.185","adjustedEnd":"0:01:25.218","start":"0:01:25.185","end":"0:01:25.218"}]},{"id":34,"instances":[{"thumbnailId":"961ccc1f-719b-4804-a097-e7f8cb826b19","adjustedStart":"0:01:25.619","adjustedEnd":"0:01:25.652","start":"0:01:25.619","end":"0:01:25.652"}]},{"id":35,"instances":[{"thumbnailId":"4503ba5f-d1ef-44ff-bc6b-6d5c61e72dbf","adjustedStart":"0:01:26.153","adjustedEnd":"0:01:26.186","start":"0:01:26.153","end":"0:01:26.186"}]}],"instances":[{"adjustedStart":"0:01:25.185","adjustedEnd":"0:01:26.62","start":"0:01:25.185","end":"0:01:26.62"}]},{"id":13,"tags":["Indoor"],"keyFrames":[{"id":36,"instances":[{"thumbnailId":"46678bbc-6c5a-4057-bf79-96edc31acd74","adjustedStart":"0:01:26.62","adjustedEnd":"0:01:26.653","start":"0:01:26.62","end":"0:01:26.653"}]},{"id":37,"instances":[{"thumbnailId":"98a6a5c7-dc5e-4b4f-a1eb-4717156c063e","adjustedStart":"0:01:27.854","adjustedEnd":"0:01:27.887","start":"0:01:27.854","end":"0:01:27.887"}]},{"id":38,"instances":[{"thumbnailId":"bd977978-fe0f-4d5a-bd56-b4a90317d32d","adjustedStart":"0:01:29.623","adjustedEnd":"0:01:29.656","start":"0:01:29.623","end":"0:01:29.656"}]}],"instances":[{"adjustedStart":"0:01:26.62","adjustedEnd":"0:01:30.09","start":"0:01:26.62","end":"0:01:30.09"}]},{"id":14,"tags":["Outdoor"],"keyFrames":[{"id":39,"instances":[{"thumbnailId":"d2e23245-da75-40ac-ae99-647dc7400550","adjustedStart":"0:01:30.09","adjustedEnd":"0:01:30.123","start":"0:01:30.09","end":"0:01:30.123"}]},{"id":40,"instances":[{"thumbnailId":"9d0ec3e8-52fd-4889-a09b-57478bf447f9","adjustedStart":"0:01:33.293","adjustedEnd":"0:01:33.326","start":"0:01:33.293","end":"0:01:33.326"}]},{"id":41,"instances":[{"thumbnailId":"f906017d-7b40-48d1-9464-649578d8baff","adjustedStart":"0:01:37.598","adjustedEnd":"0:01:37.631","start":"0:01:37.598","end":"0:01:37.631"}]}],"instances":[{"adjustedStart":"0:01:30.09","adjustedEnd":"0:01:37.798","start":"0:01:30.09","end":"0:01:37.798"}]},{"id":15,"tags":["Outdoor","Indoor"],"keyFrames":[{"id":42,"instances":[{"thumbnailId":"8c96f6bc-de5d-48b6-b6df-fb23a761c95d","adjustedStart":"0:01:37.798","adjustedEnd":"0:01:37.831","start":"0:01:37.798","end":"0:01:37.831"}]},{"id":43,"instances":[{"thumbnailId":"f66962b8-11d5-4f30-b6cc-791a84e9e386","adjustedStart":"0:01:39.866","adjustedEnd":"0:01:39.899","start":"0:01:39.866","end":"0:01:39.899"}]}],"instances":[{"adjustedStart":"0:01:37.798","adjustedEnd":"0:01:43.003","start":"0:01:37.798","end":"0:01:43.003"}]},{"id":16,"tags":["Indoor"],"keyFrames":[{"id":44,"instances":[{"thumbnailId":"8777fcc0-6ad3-4395-bfc9-7e186d3ab9f3","adjustedStart":"0:01:43.003","adjustedEnd":"0:01:43.036","start":"0:01:43.003","end":"0:01:43.036"}]},{"id":45,"instances":[{"thumbnailId":"1873dfd1-9af0-4882-80da-4ede5bea62ca","adjustedStart":"0:01:47.307","adjustedEnd":"0:01:47.34","start":"0:01:47.307","end":"0:01:47.34"}]},{"id":46,"instances":[{"thumbnailId":"49082bac-f730-4853-8f27-11072832c24c","adjustedStart":"0:01:50.611","adjustedEnd":"0:01:50.644","start":"0:01:50.611","end":"0:01:50.644"}]}],"instances":[{"adjustedStart":"0:01:43.003","adjustedEnd":"0:01:50.744","start":"0:01:43.003","end":"0:01:50.744"}]},{"id":17,"tags":["Indoor"],"keyFrames":[{"id":47,"instances":[{"thumbnailId":"df563308-3b64-4998-ad8d-3cf46d733f63","adjustedStart":"0:01:50.744","adjustedEnd":"0:01:50.777","start":"0:01:50.744","end":"0:01:50.777"}]},{"id":48,"instances":[{"thumbnailId":"88bb0da5-f046-4884-92d5-b45b719e7e8f","adjustedStart":"0:01:55.449","adjustedEnd":"0:01:55.482","start":"0:01:55.449","end":"0:01:55.482"}]},{"id":49,"instances":[{"thumbnailId":"07c2fc4b-962c-4ad1-abef-9c0cfb7a0030","adjustedStart":"0:01:58.285","adjustedEnd":"0:01:58.318","start":"0:01:58.285","end":"0:01:58.318"}]}],"instances":[{"adjustedStart":"0:01:50.744","adjustedEnd":"0:02:00.454","start":"0:01:50.744","end":"0:02:00.454"}]},{"id":18,"tags":["Indoor"],"keyFrames":[{"id":50,"instances":[{"thumbnailId":"a777a7e9-1707-437c-be32-a08b5265d1d3","adjustedStart":"0:02:00.454","adjustedEnd":"0:02:00.487","start":"0:02:00.454","end":"0:02:00.487"}]},{"id":51,"instances":[{"thumbnailId":"48930358-311d-4ad6-a4c4-551480e4c8c7","adjustedStart":"0:02:04.224","adjustedEnd":"0:02:04.257","start":"0:02:04.224","end":"0:02:04.257"}]},{"id":52,"instances":[{"thumbnailId":"b221f426-01e6-4a38-98a7-537084368029","adjustedStart":"0:02:05.926","adjustedEnd":"0:02:05.959","start":"0:02:05.926","end":"0:02:05.959"}]}],"instances":[{"adjustedStart":"0:02:00.454","adjustedEnd":"0:02:09.83","start":"0:02:00.454","end":"0:02:09.83"}]},{"id":19,"tags":["Indoor"],"keyFrames":[{"id":53,"instances":[{"thumbnailId":"75dcf761-99c0-4253-ae1e-1aec124fc515","adjustedStart":"0:02:09.83","adjustedEnd":"0:02:09.863","start":"0:02:09.83","end":"0:02:09.863"}]},{"id":54,"instances":[{"thumbnailId":"3be08d54-7d24-4a2d-b1d2-3dea6fd1ade9","adjustedStart":"0:02:12.899","adjustedEnd":"0:02:12.932","start":"0:02:12.899","end":"0:02:12.932"}]},{"id":55,"instances":[{"thumbnailId":"cdb79929-c022-45c3-a372-838915ac4289","adjustedStart":"0:02:14.768","adjustedEnd":"0:02:14.801","start":"0:02:14.768","end":"0:02:14.801"}]}],"instances":[{"adjustedStart":"0:02:09.83","adjustedEnd":"0:02:17.738","start":"0:02:09.83","end":"0:02:17.738"}]},{"id":20,"tags":["Indoor"],"keyFrames":[{"id":56,"instances":[{"thumbnailId":"c0c35836-860e-4b34-8c28-553868a237fa","adjustedStart":"0:02:17.738","adjustedEnd":"0:02:17.771","start":"0:02:17.738","end":"0:02:17.771"}]},{"id":57,"instances":[{"thumbnailId":"f034b40d-1142-4d1f-ba3b-7fd5db905042","adjustedStart":"0:02:20.107","adjustedEnd":"0:02:20.14","start":"0:02:20.107","end":"0:02:20.14"}]},{"id":58,"instances":[{"thumbnailId":"38bf4446-4b23-403c-9bfb-6a5218180766","adjustedStart":"0:02:21.341","adjustedEnd":"0:02:21.374","start":"0:02:21.341","end":"0:02:21.374"}]}],"instances":[{"adjustedStart":"0:02:17.738","adjustedEnd":"0:02:21.441","start":"0:02:17.738","end":"0:02:21.441"}]},{"id":21,"tags":["Indoor"],"keyFrames":[{"id":59,"instances":[{"thumbnailId":"3057ea31-1c1d-4a44-989a-3beba6a9359b","adjustedStart":"0:02:21.441","adjustedEnd":"0:02:21.474","start":"0:02:21.441","end":"0:02:21.474"}]},{"id":60,"instances":[{"thumbnailId":"0ba20b9e-7076-424c-a3d8-2f17a7c27024","adjustedStart":"0:02:24.578","adjustedEnd":"0:02:24.611","start":"0:02:24.578","end":"0:02:24.611"}]},{"id":61,"instances":[{"thumbnailId":"0a1730de-fe71-4845-8b34-6fb118c16b23","adjustedStart":"0:02:27.447","adjustedEnd":"0:02:27.48","start":"0:02:27.447","end":"0:02:27.48"}]}],"instances":[{"adjustedStart":"0:02:21.441","adjustedEnd":"0:02:27.614","start":"0:02:21.441","end":"0:02:27.614"}]},{"id":22,"tags":["Indoor"],"keyFrames":[{"id":62,"instances":[{"thumbnailId":"76e55ef0-9d77-4e82-b56f-5dc2e2b6db12","adjustedStart":"0:02:27.614","adjustedEnd":"0:02:27.647","start":"0:02:27.614","end":"0:02:27.647"}]},{"id":63,"instances":[{"thumbnailId":"8f78ef50-1f19-46a4-93da-f87901c23a00","adjustedStart":"0:02:27.748","adjustedEnd":"0:02:27.781","start":"0:02:27.748","end":"0:02:27.781"}]}],"instances":[{"adjustedStart":"0:02:27.614","adjustedEnd":"0:02:31.818","start":"0:02:27.614","end":"0:02:31.818"}]},{"id":23,"tags":["Indoor"],"keyFrames":[{"id":64,"instances":[{"thumbnailId":"1c1eacbc-ab30-4d0f-9bce-4f3ec77bbf88","adjustedStart":"0:02:31.818","adjustedEnd":"0:02:31.851","start":"0:02:31.818","end":"0:02:31.851"}]},{"id":65,"instances":[{"thumbnailId":"79bfd9b4-1f73-40bb-a918-564429d8adf3","adjustedStart":"0:02:34.554","adjustedEnd":"0:02:34.587","start":"0:02:34.554","end":"0:02:34.587"}]}],"instances":[{"adjustedStart":"0:02:31.818","adjustedEnd":"0:02:38.125","start":"0:02:31.818","end":"0:02:38.125"}]},{"id":24,"tags":["Indoor"],"keyFrames":[{"id":66,"instances":[{"thumbnailId":"0bc3ed31-7e77-4a8a-8419-7adf56634eb0","adjustedStart":"0:02:38.125","adjustedEnd":"0:02:38.158","start":"0:02:38.125","end":"0:02:38.158"}]},{"id":67,"instances":[{"thumbnailId":"87df53ba-69e0-4b17-a034-153d0a3faebe","adjustedStart":"0:02:39.526","adjustedEnd":"0:02:39.559","start":"0:02:39.526","end":"0:02:39.559"}]}],"instances":[{"adjustedStart":"0:02:38.125","adjustedEnd":"0:02:41.495","start":"0:02:38.125","end":"0:02:41.495"}]},{"id":25,"tags":["Indoor"],"keyFrames":[{"id":68,"instances":[{"thumbnailId":"b0a86447-87cc-420b-b3bd-9c1bbf1dcf11","adjustedStart":"0:02:41.495","adjustedEnd":"0:02:41.528","start":"0:02:41.495","end":"0:02:41.528"}]},{"id":69,"instances":[{"thumbnailId":"94da8210-0dc2-47fe-806a-8230f9921ce9","adjustedStart":"0:02:43.163","adjustedEnd":"0:02:43.196","start":"0:02:43.163","end":"0:02:43.196"}]},{"id":70,"instances":[{"thumbnailId":"9f595b79-7a68-4aa7-9fbf-1d2f34de1548","adjustedStart":"0:02:43.597","adjustedEnd":"0:02:43.63","start":"0:02:43.597","end":"0:02:43.63"}]}],"instances":[{"adjustedStart":"0:02:41.495","adjustedEnd":"0:02:45.666","start":"0:02:41.495","end":"0:02:45.666"}]},{"id":26,"tags":["Indoor"],"keyFrames":[{"id":71,"instances":[{"thumbnailId":"e11b2fd8-9b9d-4733-9433-6eeaedd56624","adjustedStart":"0:02:45.666","adjustedEnd":"0:02:45.699","start":"0:02:45.666","end":"0:02:45.699"}]},{"id":72,"instances":[{"thumbnailId":"ecff645e-69cc-4b5c-8d32-810c8fdd6994","adjustedStart":"0:02:46.967","adjustedEnd":"0:02:47","start":"0:02:46.967","end":"0:02:47"}]},{"id":73,"instances":[{"thumbnailId":"3c121573-8e98-4f1d-bfc5-979df29c6f46","adjustedStart":"0:02:49.603","adjustedEnd":"0:02:49.636","start":"0:02:49.603","end":"0:02:49.636"}]}],"instances":[{"adjustedStart":"0:02:45.666","adjustedEnd":"0:02:51.238","start":"0:02:45.666","end":"0:02:51.238"}]},{"id":27,"tags":["Indoor"],"keyFrames":[{"id":74,"instances":[{"thumbnailId":"0abac2de-f2b8-448f-a4ea-51c0e759d8bd","adjustedStart":"0:02:51.238","adjustedEnd":"0:02:51.271","start":"0:02:51.238","end":"0:02:51.271"}]},{"id":75,"instances":[{"thumbnailId":"c721b5ad-4f26-4382-83ab-e0d9dccede35","adjustedStart":"0:02:52.639","adjustedEnd":"0:02:52.672","start":"0:02:52.639","end":"0:02:52.672"}]},{"id":76,"instances":[{"thumbnailId":"33f4b1b6-962f-4314-8651-5b4a90ee5f11","adjustedStart":"0:02:54.474","adjustedEnd":"0:02:54.507","start":"0:02:54.474","end":"0:02:54.507"}]}],"instances":[{"adjustedStart":"0:02:51.238","adjustedEnd":"0:02:55.142","start":"0:02:51.238","end":"0:02:55.142"}]},{"id":28,"tags":["Indoor","ExtremeCloseUp","RightFace"],"keyFrames":[{"id":77,"instances":[{"thumbnailId":"16dd2f87-5720-4116-8399-60c7cff3317e","adjustedStart":"0:02:55.142","adjustedEnd":"0:02:55.175","start":"0:02:55.142","end":"0:02:55.175"}]},{"id":78,"instances":[{"thumbnailId":"a9d80e93-5311-4a12-8664-1091d165e0dc","adjustedStart":"0:02:56.143","adjustedEnd":"0:02:56.176","start":"0:02:56.143","end":"0:02:56.176"}]},{"id":79,"instances":[{"thumbnailId":"619dd8e7-a0f2-4a78-955d-01dec2501999","adjustedStart":"0:02:58.645","adjustedEnd":"0:02:58.678","start":"0:02:58.645","end":"0:02:58.678"}]}],"instances":[{"adjustedStart":"0:02:55.142","adjustedEnd":"0:02:58.912","start":"0:02:55.142","end":"0:02:58.912"}]},{"id":29,"tags":["ExtremeCloseUp","RightFace","Indoor"],"keyFrames":[{"id":80,"instances":[{"thumbnailId":"8c06bb05-578c-435c-a734-b54f0b6542ac","adjustedStart":"0:02:58.912","adjustedEnd":"0:02:58.945","start":"0:02:58.912","end":"0:02:58.945"}]},{"id":81,"instances":[{"thumbnailId":"fdfdeee3-e1a3-4723-9a81-374e4e285ed1","adjustedStart":"0:03:00.38","adjustedEnd":"0:03:00.413","start":"0:03:00.38","end":"0:03:00.413"}]},{"id":82,"instances":[{"thumbnailId":"a9616fdd-b8f7-4460-90d8-e514281fc835","adjustedStart":"0:03:01.014","adjustedEnd":"0:03:01.047","start":"0:03:01.014","end":"0:03:01.047"}]}],"instances":[{"adjustedStart":"0:02:58.912","adjustedEnd":"0:03:01.381","start":"0:02:58.912","end":"0:03:01.381"}]},{"id":30,"tags":["Indoor"],"keyFrames":[{"id":83,"instances":[{"thumbnailId":"a6c97079-db4f-42db-a5a0-407be4bbf5cd","adjustedStart":"0:03:01.381","adjustedEnd":"0:03:01.414","start":"0:03:01.381","end":"0:03:01.414"}]},{"id":84,"instances":[{"thumbnailId":"417c3ebe-ff78-4d03-a22a-72ea183bf473","adjustedStart":"0:03:01.915","adjustedEnd":"0:03:01.948","start":"0:03:01.915","end":"0:03:01.948"}]},{"id":85,"instances":[{"thumbnailId":"bb2d66e0-004f-4023-9c06-907352d4d718","adjustedStart":"0:03:02.816","adjustedEnd":"0:03:02.849","start":"0:03:02.816","end":"0:03:02.849"}]}],"instances":[{"adjustedStart":"0:03:01.381","adjustedEnd":"0:03:03.083","start":"0:03:01.381","end":"0:03:03.083"}]},{"id":31,"tags":["Indoor"],"keyFrames":[{"id":86,"instances":[{"thumbnailId":"549d5d14-bd81-4634-abdb-149e359d2651","adjustedStart":"0:03:03.083","adjustedEnd":"0:03:03.116","start":"0:03:03.083","end":"0:03:03.116"}]},{"id":87,"instances":[{"thumbnailId":"b4885f6e-5e78-4561-8e04-91d90c37d982","adjustedStart":"0:03:06.119","adjustedEnd":"0:03:06.152","start":"0:03:06.119","end":"0:03:06.152"}]},{"id":88,"instances":[{"thumbnailId":"cb7a5e74-ae03-43e7-893b-e5e88cb5a162","adjustedStart":"0:03:08.522","adjustedEnd":"0:03:08.555","start":"0:03:08.522","end":"0:03:08.555"}]}],"instances":[{"adjustedStart":"0:03:03.083","adjustedEnd":"0:03:08.789","start":"0:03:03.083","end":"0:03:08.789"}]},{"id":32,"tags":["Indoor"],"keyFrames":[{"id":89,"instances":[{"thumbnailId":"1b97c14d-8aa8-477d-9da5-94da7e3dd518","adjustedStart":"0:03:08.789","adjustedEnd":"0:03:08.822","start":"0:03:08.789","end":"0:03:08.822"}]},{"id":90,"instances":[{"thumbnailId":"514f0620-86c8-48b3-9105-9a5adec35996","adjustedStart":"0:03:10.891","adjustedEnd":"0:03:10.924","start":"0:03:10.891","end":"0:03:10.924"}]},{"id":91,"instances":[{"thumbnailId":"3c411768-2785-4ba4-9f29-971cbb6e071d","adjustedStart":"0:03:12.259","adjustedEnd":"0:03:12.292","start":"0:03:12.259","end":"0:03:12.292"}]}],"instances":[{"adjustedStart":"0:03:08.789","adjustedEnd":"0:03:12.426","start":"0:03:08.789","end":"0:03:12.426"}]},{"id":33,"tags":["Indoor"],"keyFrames":[{"id":92,"instances":[{"thumbnailId":"604b8d52-f0ea-4934-85b6-9e8e8cda65f8","adjustedStart":"0:03:12.426","adjustedEnd":"0:03:12.459","start":"0:03:12.426","end":"0:03:12.459"}]},{"id":93,"instances":[{"thumbnailId":"4b4ed089-507c-462f-8e30-18c8c18670ee","adjustedStart":"0:03:14.594","adjustedEnd":"0:03:14.627","start":"0:03:14.594","end":"0:03:14.627"}]},{"id":94,"instances":[{"thumbnailId":"4636ca4b-b36f-49ac-a668-e9936a6d8dbe","adjustedStart":"0:03:15.696","adjustedEnd":"0:03:15.729","start":"0:03:15.696","end":"0:03:15.729"}]}],"instances":[{"adjustedStart":"0:03:12.426","adjustedEnd":"0:03:15.796","start":"0:03:12.426","end":"0:03:15.796"}]},{"id":34,"tags":["Indoor"],"keyFrames":[{"id":95,"instances":[{"thumbnailId":"25d65088-5215-4495-9059-eec23d2b58e5","adjustedStart":"0:03:15.796","adjustedEnd":"0:03:15.829","start":"0:03:15.796","end":"0:03:15.829"}]},{"id":96,"instances":[{"thumbnailId":"9f463ceb-1c75-439d-a870-3047354a0df8","adjustedStart":"0:03:16.263","adjustedEnd":"0:03:16.296","start":"0:03:16.263","end":"0:03:16.296"}]},{"id":97,"instances":[{"thumbnailId":"169cf866-f1bd-4c66-ba4b-df37c392ff37","adjustedStart":"0:03:19.599","adjustedEnd":"0:03:19.632","start":"0:03:19.599","end":"0:03:19.632"}]}],"instances":[{"adjustedStart":"0:03:15.796","adjustedEnd":"0:03:19.666","start":"0:03:15.796","end":"0:03:19.666"}]},{"id":35,"tags":["Indoor"],"keyFrames":[{"id":98,"instances":[{"thumbnailId":"10dd8e67-b6a9-4393-8e27-a8010d1f9e0d","adjustedStart":"0:03:19.666","adjustedEnd":"0:03:19.699","start":"0:03:19.666","end":"0:03:19.699"}]},{"id":99,"instances":[{"thumbnailId":"88b57fec-30b4-45f0-a459-ca68ae78e70c","adjustedStart":"0:03:23.704","adjustedEnd":"0:03:23.737","start":"0:03:23.704","end":"0:03:23.737"}]}],"instances":[{"adjustedStart":"0:03:19.666","adjustedEnd":"0:03:26.406","start":"0:03:19.666","end":"0:03:26.406"}]},{"id":36,"tags":["Indoor"],"keyFrames":[{"id":100,"instances":[{"thumbnailId":"38733f8e-47df-4196-ae19-84afa3edd1db","adjustedStart":"0:03:26.406","adjustedEnd":"0:03:26.439","start":"0:03:26.406","end":"0:03:26.439"}]},{"id":101,"instances":[{"thumbnailId":"92ec4cca-badd-47da-b24c-1ebce4e7cc54","adjustedStart":"0:03:26.707","adjustedEnd":"0:03:26.74","start":"0:03:26.707","end":"0:03:26.74"}]},{"id":102,"instances":[{"thumbnailId":"c1e17124-2a97-4684-b7cf-a6452d5bcafc","adjustedStart":"0:03:27.374","adjustedEnd":"0:03:27.407","start":"0:03:27.374","end":"0:03:27.407"}]}],"instances":[{"adjustedStart":"0:03:26.406","adjustedEnd":"0:03:27.974","start":"0:03:26.406","end":"0:03:27.974"}]},{"id":37,"keyFrames":[{"id":103,"instances":[{"thumbnailId":"7acf2b26-28ba-4289-a3d7-82b491d165c0","adjustedStart":"0:03:27.974","adjustedEnd":"0:03:28.007","start":"0:03:27.974","end":"0:03:28.007"}]},{"id":104,"instances":[{"thumbnailId":"d65469b3-1364-41ab-ae30-bb031359491c","adjustedStart":"0:03:29.176","adjustedEnd":"0:03:29.209","start":"0:03:29.176","end":"0:03:29.209"}]},{"id":105,"instances":[{"thumbnailId":"96118608-5170-476a-8ba7-3fd454b35fa6","adjustedStart":"0:03:31.712","adjustedEnd":"0:03:31.745","start":"0:03:31.712","end":"0:03:31.745"}]}],"instances":[{"adjustedStart":"0:03:27.974","adjustedEnd":"0:03:31.845","start":"0:03:27.974","end":"0:03:31.845"}]},{"id":38,"keyFrames":[{"id":106,"instances":[{"thumbnailId":"e27374e9-3171-4008-be6a-9743bd8c24ab","adjustedStart":"0:03:31.845","adjustedEnd":"0:03:31.878","start":"0:03:31.845","end":"0:03:31.878"}]}],"instances":[{"adjustedStart":"0:03:31.845","adjustedEnd":"0:03:31.978","start":"0:03:31.845","end":"0:03:31.978"}]}],"sentiments":[{"id":1,"averageScore":0.5,"sentimentType":"Neutral","instances":[{"adjustedStart":"0:00:00","adjustedEnd":"0:00:31.95","start":"0:00:00","end":"0:00:31.95"},{"adjustedStart":"0:00:37.25","adjustedEnd":"0:03:32.045","start":"0:00:37.25","end":"0:03:32.045"}]},{"id":2,"averageScore":0.1819,"sentimentType":"Negative","instances":[{"adjustedStart":"0:00:31.95","adjustedEnd":"0:00:37.25","start":"0:00:31.95","end":"0:00:37.25"}]}],"emotions":[{"id":1,"type":"Sad","instances":[{"confidence":0.9097,"adjustedStart":"0:00:31.95","adjustedEnd":"0:00:37.25","start":"0:00:31.95","end":"0:00:37.25"}]}],"visualContentModeration":[{"id":1,"adultScore":0.9442,"racyScore":0.0512,"instances":[{"adjustedStart":"0:00:15.115","adjustedEnd":"0:00:15.148","start":"0:00:15.115","end":"0:00:15.148"}]},{"id":2,"adultScore":0.0006,"racyScore":0.897,"instances":[{"adjustedStart":"0:01:43.003","adjustedEnd":"0:01:43.036","start":"0:01:43.003","end":"0:01:43.036"}]},{"id":3,"adultScore":0.1067,"racyScore":0.8224,"instances":[{"adjustedStart":"0:01:55.449","adjustedEnd":"0:01:55.616","start":"0:01:55.449","end":"0:01:55.616"}]},{"id":4,"adultScore":0.0002,"racyScore":0.8415,"instances":[{"adjustedStart":"0:02:00.454","adjustedEnd":"0:02:00.487","start":"0:02:00.454","end":"0:02:00.487"}]},{"id":5,"adultScore":0.991,"racyScore":0.0029,"instances":[{"adjustedStart":"0:02:09.83","adjustedEnd":"0:02:09.863","start":"0:02:09.83","end":"0:02:09.863"}]},{"id":6,"adultScore":0.004,"racyScore":0.8639,"instances":[{"adjustedStart":"0:03:12.426","adjustedEnd":"0:03:12.459","start":"0:03:12.426","end":"0:03:12.459"}]},{"id":7,"adultScore":0.0019,"racyScore":0.829,"instances":[{"adjustedStart":"0:03:15.696","adjustedEnd":"0:03:15.729","start":"0:03:15.696","end":"0:03:15.729"}]}],"blocks":[{"id":0,"instances":[{"adjustedStart":"0:00:00","adjustedEnd":"0:00:31.95","start":"0:00:00","end":"0:00:31.95"}]},{"id":1,"instances":[{"adjustedStart":"0:00:31.95","adjustedEnd":"0:01:01.96","start":"0:00:31.95","end":"0:01:01.96"}]},{"id":2,"instances":[{"adjustedStart":"0:01:01.96","adjustedEnd":"0:01:43.92","start":"0:01:01.96","end":"0:01:43.92"}]},{"id":3,"instances":[{"adjustedStart":"0:01:43.92","adjustedEnd":"0:02:26.2","start":"0:01:43.92","end":"0:02:26.2"}]},{"id":4,"instances":[{"adjustedStart":"0:02:26.2","adjustedEnd":"0:02:56.62","start":"0:02:26.2","end":"0:02:56.62"}]},{"id":5,"instances":[{"adjustedStart":"0:02:56.62","adjustedEnd":"0:03:32.045","start":"0:02:56.62","end":"0:03:32.045"}]}],"framePatterns":[{"id":1,"patternType":"RollingCredits","confidence":0.1,"instances":[{"adjustedStart":"0:03:27","adjustedEnd":"0:03:32","start":"0:03:27","end":"0:03:32"}]},{"id":2,"patternType":"Black","confidence":1,"instances":[{"adjustedStart":"0:00:00","adjustedEnd":"0:00:00.334","start":"0:00:00","end":"0:00:00.334"},{"adjustedStart":"0:03:31.845","adjustedEnd":"0:03:31.945","start":"0:03:31.845","end":"0:03:31.945"}]}],"speakers":[{"id":1,"name":"Speaker #1","instances":[{"adjustedStart":"0:00:07.82","adjustedEnd":"0:00:11.66","start":"0:00:07.82","end":"0:00:11.66"}]},{"id":2,"name":"Speaker #2","instances":[{"adjustedStart":"0:00:11.66","adjustedEnd":"0:00:25.2","start":"0:00:11.66","end":"0:00:25.2"}]},{"id":4,"name":"Speaker #4","instances":[{"adjustedStart":"0:00:25.2","adjustedEnd":"0:00:42.15","start":"0:00:25.2","end":"0:00:42.15"}]},{"id":5,"name":"Speaker #5","instances":[{"adjustedStart":"0:00:42.15","adjustedEnd":"0:00:52.62","start":"0:00:42.15","end":"0:00:52.62"}]},{"id":6,"name":"Speaker #6","instances":[{"adjustedStart":"0:00:52.62","adjustedEnd":"0:01:03.62","start":"0:00:52.62","end":"0:01:03.62"}]},{"id":3,"name":"Speaker #3","instances":[{"adjustedStart":"0:01:03.62","adjustedEnd":"0:01:16.67","start":"0:01:03.62","end":"0:01:16.67"}]},{"id":7,"name":"Speaker #7","instances":[{"adjustedStart":"0:01:16.67","adjustedEnd":"0:01:30.34","start":"0:01:16.67","end":"0:01:30.34"}]},{"id":8,"name":"Speaker #8","instances":[{"adjustedStart":"0:01:36.88","adjustedEnd":"0:01:43.92","start":"0:01:36.88","end":"0:01:43.92"}]},{"id":9,"name":"Speaker #9","instances":[{"adjustedStart":"0:01:44.68","adjustedEnd":"0:01:57.13","start":"0:01:44.68","end":"0:01:57.13"}]},{"id":11,"name":"Speaker #11","instances":[{"adjustedStart":"0:01:57.13","adjustedEnd":"0:02:01.76","start":"0:01:57.13","end":"0:02:01.76"},{"adjustedStart":"0:02:01.884","adjustedEnd":"0:02:12.6","start":"0:02:01.884","end":"0:02:12.6"}]},{"id":10,"name":"Speaker #10","instances":[{"adjustedStart":"0:02:12.6","adjustedEnd":"0:02:26.2","start":"0:02:12.6","end":"0:02:26.2"}]},{"id":12,"name":"Speaker #12","instances":[{"adjustedStart":"0:02:39.9","adjustedEnd":"0:02:56.62","start":"0:02:39.9","end":"0:02:56.62"}]},{"id":13,"name":"Speaker #13","instances":[{"adjustedStart":"0:02:59.5","adjustedEnd":"0:03:04.27","start":"0:02:59.5","end":"0:03:04.27"}]}],"textualContentModeration":{"id":0,"bannedWordsCount":0,"bannedWordsRatio":0,"instances":[]},"statistics":{"correspondenceCount":12,"speakerTalkToListenRatio":{"1":0.025,"2":0.088,"3":0.085,"4":0.111,"5":0.068,"6":0.072,"7":0.089,"8":0.046,"9":0.081,"10":0.089,"11":0.1,"12":0.109,"13":0.031},"speakerLongestMonolog":{"1":3,"2":13,"3":13,"4":16,"5":10,"6":11,"7":13,"8":7,"9":12,"10":13,"11":15,"12":16,"13":4},"speakerNumberOfFragments":{"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1,"10":1,"11":2,"12":1,"13":1},"speakerWordCount":{"1":3,"2":14,"3":18,"4":35,"5":9,"6":8,"7":7,"8":2,"9":13,"10":4,"11":11,"12":3,"13":2}}},"thumbnailId":"2fedb27e-6f45-4dca-947a-d850c10722d4","detectSourceLanguage":false,"languageAutoDetectMode":"None","sourceLanguage":"en-US","sourceLanguages":["en-US"],"language":"en-US","languages":["en-US"],"indexingPreset":"Default","linguisticModelId":"00000000-0000-0000-0000-000000000000","personModelId":"00000000-0000-0000-0000-000000000000","isAdult":false,"publishedUrl":"https://rodmandev.streaming.mediaservices.windows.net/a1d07433-752e-4097-b266-c23e67e9f8e6/e1af7563-0b4b-4e8c-953f-898f8548.ism/manifest(encryption=cbc)","publishedProxyUrl":null,"viewToken":"Bearer=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1cm46bWljcm9zb2Z0OmF6dXJlOm1lZGlhc2VydmljZXM6Y29udGVudGtleWlkZW50aWZpZXIiOiIyZTI2YzdjZS1kZThjLTRlNTUtOWM3Ni1kZWRkZmM5MzhhODgiLCJuYmYiOjE1ODcxNjQ5NzksImV4cCI6MTU4NzIwODIzOSwiaXNzIjoiaHR0cHM6Ly9icmVha2Rvd24ubWUiLCJhdWQiOiJCcmVha2Rvd25Vc2VyIn0.kgTXkoSeRTslIFCZNgdPMf4pCfyDjVtKj_mMwJGp4RU"}],"videosRanges":[{"videoId":"3b83f41144","range":{"start":"0:00:00","end":"0:03:32.096"}}]} \ No newline at end of file diff --git a/controllers/insertIntoDescriptions.js b/controllers/insertIntoDescriptions.js new file mode 100644 index 0000000..7982f24 --- /dev/null +++ b/controllers/insertIntoDescriptions.js @@ -0,0 +1,121 @@ +const axios = require('axios'); +const db = require('../db/database'); +const Utils = require('../backend/utils'); + + +class insertIntoDescriptions{ + + async run(fromUserId, toUserId, videoId){ + + try { + let sceneData = await db.any(`SELECT * FROM descriptions WHERE video_id = $1 AND volunteer_id = $2 ORDER BY scene_num`, [videoId, fromUserId]); + console.log({sceneData}); + + let sql = ''; + for(let i=0; i { + console.log("Response is: ",res.data); + new_description = res.data['result']; + + if(ocr.length > 0){ + let ocr_url = 'http://3.15.220.168:5000/' + ocr; + + axios.get(ocr_url) + .then(ocr_response => { + console.log("OCR Response is: ",ocr_response.data); + new_ocr = ocr_response.data['result']; + sql = ` UPDATE descriptions SET description = '${new_description}', ocr = '${new_ocr}' WHERE volunteer_id = '${userId}' AND video_id = '${videoId}' AND desc_id = '${desc_id}'; ` + let sql_commit = 'START TRANSACTION;' + sql + 'COMMIT;'; + console.log({sql_commit}); + + let res = db.query(sql_commit); + console.log(res); + }) + .catch(error => { + console.log(error); + }); + + + } + else{ + sql = ` UPDATE descriptions SET description = '${new_description}' WHERE volunteer_id = '${userId}' AND video_id = '${videoId}' AND desc_id = '${desc_id}'; ` + let sql_commit = 'START TRANSACTION;' + sql + 'COMMIT;'; + console.log({sql_commit}); + + let res = db.query(sql_commit); + console.log(res); + } + + + }) + .catch(error => { + console.log(error); + }); + + console.log({new_ocr}); + console.log({new_description}); + + // sql += ` UPDATE descriptions SET description = '${new_description}', ocr = '${new_ocr}' WHERE volunteer_id = '${userId}' AND video_id = '${videoId}' AND description_id = '${descriptionId}'; ` + + + } + + sql = 'START TRANSACTION;' + sql + 'COMMIT;'; + console.log({sql}); + + // let res = await db.query(sql); + // console.log(res); + + }catch(e){ + console.log(e); + throw e; + } + } + +} + +let cont = new insertIntoDescriptions(); +// cont.run('5117b1e0-4c45-11ea-b77f-2e728ce88125', '9fe1e2c6-5cd2-11ea-bc55-0242ac130003', '7QZNpS0uos4'); +// cont.start('d8fcfd52-b0b3-4969-bf03-1dc54b681b5d', 'Q9Xp77AENXI'); \ No newline at end of file diff --git a/controllers/insight.json b/controllers/insight.json new file mode 100644 index 0000000..7456d1a --- /dev/null +++ b/controllers/insight.json @@ -0,0 +1 @@ +{"partition":null,"description":null,"privacyMode":"Public","state":"Processed","accountId":"5a8c6ddf-900f-48b4-8a3a-0cd5cf48cb4b","id":"2a715d49a3","name":"Ocean's 8 (2018) - Con Artistry Scene (1_10) _ Movieclips-OkBaZLq7gnU","userName":"Umang Mathur","created":"2020-02-10T07:52:05.4961675+00:00","isOwned":false,"isEditable":false,"isBase":true,"durationInSeconds":152,"summarizedInsights":{"name":"Ocean's 8 (2018) - Con Artistry Scene (1_10) _ Movieclips-OkBaZLq7gnU","id":"2a715d49a3","privacyMode":"Public","duration":{"time":"0:02:32.512","seconds":152.5},"thumbnailVideoId":"2a715d49a3","thumbnailId":"6aa5e2d7-6c19-4084-9090-739fb76e1b3a","faces":[{"videoId":"2a715d49a3","confidence":0,"description":null,"title":null,"thumbnailId":"6dfe2efd-4caf-48ce-b144-10f6a4c10f7e","seenDuration":104.8,"seenDurationRatio":0.6872,"id":1003,"name":"Unknown #1","appearances":[{"startTime":"0:00:05.589","endTime":"0:00:56.181","startSeconds":5.6,"endSeconds":56.2},{"startTime":"0:01:24.585","endTime":"0:02:18.847","startSeconds":84.6,"endSeconds":138.8}]},{"videoId":"2a715d49a3","confidence":0,"description":null,"title":null,"thumbnailId":"89853eee-7f72-451c-96fb-3a38cc8f3a34","seenDuration":14.2,"seenDurationRatio":0.0931,"id":1336,"name":"Unknown #2","appearances":[{"startTime":"0:00:34.701","endTime":"0:00:48.924","startSeconds":34.7,"endSeconds":48.9}]},{"videoId":"2a715d49a3","confidence":0,"description":null,"title":null,"thumbnailId":"6e01e0bf-abf4-4393-adb5-a9d7cf9aae4a","seenDuration":3.3,"seenDurationRatio":0.0216,"id":1586,"name":"Unknown #4","appearances":[{"startTime":"0:01:51.028","endTime":"0:01:52.78","startSeconds":111,"endSeconds":112.8},{"startTime":"0:02:20.891","endTime":"0:02:22.434","startSeconds":140.9,"endSeconds":142.4}]},{"videoId":"2a715d49a3","confidence":0,"description":null,"title":null,"thumbnailId":"73964295-dfd9-4fea-8540-853126bb64f9","seenDuration":2.3,"seenDurationRatio":0.0151,"id":1083,"name":"Unknown #3","appearances":[{"startTime":"0:00:24.942","endTime":"0:00:27.152","startSeconds":24.9,"endSeconds":27.2}]},{"videoId":"2a715d49a3","confidence":0,"description":null,"title":null,"thumbnailId":"c9c8008b-80e1-44ff-a8c6-27b44b141d16","seenDuration":1.2,"seenDurationRatio":0.0079,"id":1095,"name":"Unknown #6","appearances":[{"startTime":"0:00:27.194","endTime":"0:00:28.404","startSeconds":27.2,"endSeconds":28.4}]}],"keywords":[{"isTranscript":true,"id":1,"name":"monica","appearances":[{"startTime":"0:01:10.95","endTime":"0:01:15.55","startSeconds":71,"endSeconds":75.6},{"startTime":"0:01:23.28","endTime":"0:01:31.03","startSeconds":83.3,"endSeconds":91}]},{"isTranscript":true,"id":2,"name":"back","appearances":[{"startTime":"0:01:34.9","endTime":"0:01:42.77","startSeconds":94.9,"endSeconds":102.8}]},{"isTranscript":false,"id":3,"name":"fandango","appearances":[{"startTime":"0:00:01.376","endTime":"0:00:03.212","startSeconds":1.4,"endSeconds":3.2},{"startTime":"0:02:26.355","endTime":"0:02:27.272","startSeconds":146.4,"endSeconds":147.3}]},{"isTranscript":false,"id":4,"name":"movieclips","appearances":[{"startTime":"0:00:01.376","endTime":"0:00:05.965","startSeconds":1.4,"endSeconds":6},{"startTime":"0:00:35.327","endTime":"0:00:38.08","startSeconds":35.3,"endSeconds":38.1},{"startTime":"0:02:21.308","endTime":"0:02:21.767","startSeconds":141.3,"endSeconds":141.8}]}],"sentiments":[{"sentimentKey":"Neutral","seenDurationRatio":0.9422,"appearances":[{"startTime":"0:00:00","endTime":"0:01:26.13","startSeconds":0,"endSeconds":86.1},{"startTime":"0:01:34.9","endTime":"0:02:32.462","startSeconds":94.9,"endSeconds":152.5}]},{"sentimentKey":"Negative","seenDurationRatio":0.0577,"appearances":[{"startTime":"0:01:26.13","endTime":"0:01:34.9","startSeconds":86.1,"endSeconds":94.9}]}],"emotions":[{"type":"Sad","seenDurationRatio":0.0577,"appearances":[{"startTime":"0:01:26.13","endTime":"0:01:34.9","startSeconds":86.1,"endSeconds":94.9}]}],"audioEffects":[],"labels":[{"id":1,"name":"indoor","appearances":[{"startTime":"0:00:02.669","endTime":"0:01:18.746","startSeconds":2.7,"endSeconds":78.7},{"startTime":"0:01:24.084","endTime":"0:02:22.81","startSeconds":84.1,"endSeconds":142.8}]},{"id":2,"name":"person","appearances":[{"startTime":"0:00:05.339","endTime":"0:00:57.391","startSeconds":5.3,"endSeconds":57.4},{"startTime":"0:01:01.395","endTime":"0:01:16.076","startSeconds":61.4,"endSeconds":76.1},{"startTime":"0:01:25.419","endTime":"0:01:41.435","startSeconds":85.4,"endSeconds":101.4},{"startTime":"0:01:45.439","endTime":"0:02:20.14","startSeconds":105.4,"endSeconds":140.1}]},{"id":4,"name":"wall","appearances":[{"startTime":"0:00:18.685","endTime":"0:00:20.02","startSeconds":18.7,"endSeconds":20},{"startTime":"0:00:28.028","endTime":"0:00:38.706","startSeconds":28,"endSeconds":38.7},{"startTime":"0:00:45.379","endTime":"0:00:53.387","startSeconds":45.4,"endSeconds":53.4},{"startTime":"0:01:08.068","endTime":"0:01:09.403","startSeconds":68.1,"endSeconds":69.4},{"startTime":"0:01:41.435","endTime":"0:01:45.439","startSeconds":101.4,"endSeconds":105.4},{"startTime":"0:01:54.781","endTime":"0:02:22.81","startSeconds":114.8,"endSeconds":142.8}]},{"id":3,"name":"woman","appearances":[{"startTime":"0:00:08.008","endTime":"0:00:10.678","startSeconds":8,"endSeconds":10.7},{"startTime":"0:00:28.028","endTime":"0:00:29.363","startSeconds":28,"endSeconds":29.4},{"startTime":"0:00:33.367","endTime":"0:00:54.722","startSeconds":33.4,"endSeconds":54.7},{"startTime":"0:02:01.455","endTime":"0:02:04.124","startSeconds":121.5,"endSeconds":124.1}]},{"id":6,"name":"looking","appearances":[{"startTime":"0:00:37.371","endTime":"0:00:46.714","startSeconds":37.4,"endSeconds":46.7},{"startTime":"0:01:45.439","endTime":"0:01:46.774","startSeconds":105.4,"endSeconds":106.8},{"startTime":"0:02:02.789","endTime":"0:02:04.124","startSeconds":122.8,"endSeconds":124.1}]},{"id":9,"name":"man","appearances":[{"startTime":"0:01:22.749","endTime":"0:01:24.084","startSeconds":82.7,"endSeconds":84.1},{"startTime":"0:02:06.793","endTime":"0:02:12.132","startSeconds":126.8,"endSeconds":132.1}]},{"id":5,"name":"mirror","appearances":[{"startTime":"0:00:37.371","endTime":"0:00:40.04","startSeconds":37.4,"endSeconds":40},{"startTime":"0:00:46.713","endTime":"0:00:48.048","startSeconds":46.7,"endSeconds":48}]},{"id":7,"name":"standing","appearances":[{"startTime":"0:00:50.717","endTime":"0:00:52.052","startSeconds":50.7,"endSeconds":52.1},{"startTime":"0:01:09.403","endTime":"0:01:10.738","startSeconds":69.4,"endSeconds":70.7}]},{"id":11,"name":"cat","appearances":[{"startTime":"0:02:16.136","endTime":"0:02:17.471","startSeconds":136.1,"endSeconds":137.5}]},{"id":8,"name":"laptop","appearances":[{"startTime":"0:01:14.741","endTime":"0:01:16.076","startSeconds":74.7,"endSeconds":76.1}]},{"id":10,"name":"cellphone","appearances":[{"startTime":"0:01:25.419","endTime":"0:01:26.754","startSeconds":85.4,"endSeconds":86.8}]},{"id":12,"name":"bed","appearances":[{"startTime":"0:02:21.475","endTime":"0:02:22.81","startSeconds":141.5,"endSeconds":142.8}]}],"framePatterns":[{"id":1,"name":"RollingCredits","appearances":[{"startTime":"0:02:21","endTime":"0:02:32","startSeconds":141,"endSeconds":152}]},{"id":2,"name":"Black","appearances":[{"startTime":"0:00:21.647","endTime":"0:00:21.73","startSeconds":21.6,"endSeconds":21.7}]}],"brands":[],"namedLocations":[],"namedPeople":[],"statistics":{"correspondenceCount":8,"speakerTalkToListenRatio":{"1":0.093,"2":0.039,"3":0.169,"4":0.15,"5":0.057,"6":0.37,"7":0.043,"8":0.061,"9":0.013},"speakerLongestMonolog":{"1":9,"2":3,"3":16,"4":14,"5":5,"6":36,"7":4,"8":6,"9":1},"speakerNumberOfFragments":{"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1},"speakerWordCount":{"1":1,"2":5,"3":40,"4":11,"5":11,"6":105,"7":9,"8":9,"9":2}},"topics":[{"referenceUrl":null,"iptcName":"Lifestyle and Leisure/travel and commuting","iabName":"Travel","confidence":0.3679,"id":1,"name":"Tourism/Public Transportation","appearances":[{"startTime":"0:00:00","endTime":"0:02:32.462","startSeconds":0,"endSeconds":152.5}]},{"referenceUrl":null,"iptcName":"Lifestyle and Leisure/travel and commuting","iabName":"Travel","confidence":0.2591,"id":2,"name":"Tourism/Visiting and Travel/Flights","appearances":[{"startTime":"0:00:00","endTime":"0:02:32.462","startSeconds":0,"endSeconds":152.5}]},{"referenceUrl":null,"iptcName":null,"iabName":null,"confidence":0.2074,"id":3,"name":"Aviation","appearances":[{"startTime":"0:00:00","endTime":"0:02:32.462","startSeconds":0,"endSeconds":152.5}]}]},"videos":[{"accountId":"5a8c6ddf-900f-48b4-8a3a-0cd5cf48cb4b","id":"2a715d49a3","state":"Processed","moderationState":"OK","reviewState":"None","privacyMode":"Public","processingProgress":"100%","failureCode":"None","failureMessage":"","externalId":null,"externalUrl":null,"metadata":null,"insights":{"version":"1.0.0.0","duration":"0:02:32.512","sourceLanguage":"en-US","sourceLanguages":["en-US"],"language":"en-US","languages":["en-US"],"transcript":[{"id":1,"text":"no, but they are unopened.","confidence":0.6822,"speakerId":2,"language":"en-US","instances":[{"adjustedStart":"0:00:26.33","adjustedEnd":"0:00:30.18","start":"0:00:26.33","end":"0:00:30.18"}]},{"id":2,"text":"They haven't touched I really need your receipt.","confidence":0.6822,"speakerId":3,"language":"en-US","instances":[{"adjustedStart":"0:00:30.18","adjustedEnd":"0:00:36.86","start":"0:00:30.18","end":"0:00:36.86"}]},{"id":3,"text":"They are sealed their brand new do you have the credit card that you use this is ridiculous I bought these last week?","confidence":0.8784,"speakerId":3,"language":"en-US","instances":[{"adjustedStart":"0:00:36.86","adjustedEnd":"0:00:44.08","start":"0:00:36.86","end":"0:00:44.08"}]},{"id":4,"text":"Maybe you can try client services on the six","confidence":0.8784,"speakerId":3,"language":"en-US","instances":[{"adjustedStart":"0:00:44.08","adjustedEnd":"0:00:46.91","start":"0:00:44.08","end":"0:00:46.91"}]},{"id":5,"text":"floor no never mind. And at least get a bag.","confidence":0.7482,"speakerId":4,"language":"en-US","instances":[{"adjustedStart":"0:00:46.91","adjustedEnd":"0:00:52.57","start":"0:00:46.91","end":"0:00:52.57"}]},{"id":6,"text":"Very","confidence":0.7489,"speakerId":4,"language":"en-US","instances":[{"adjustedStart":"0:00:52.57","adjustedEnd":"0:01:01.76","start":"0:00:52.57","end":"0:01:01.76"}]},{"id":7,"text":"random.","confidence":0.7489,"speakerId":1,"language":"en-US","instances":[{"adjustedStart":"0:01:01.76","adjustedEnd":"0:01:10.95","start":"0:01:01.76","end":"0:01:10.95"}]},{"id":8,"text":"Traffic. My name is Monica or anything else,","confidence":0.7962,"speakerId":5,"language":"en-US","instances":[{"adjustedStart":"0:01:10.95","adjustedEnd":"0:01:15.55","start":"0:01:10.95","end":"0:01:15.55"}]},{"id":9,"text":"I can do","confidence":0.8595,"speakerId":5,"language":"en-US","instances":[{"adjustedStart":"0:01:15.55","adjustedEnd":"0:01:16.61","start":"0:01:15.55","end":"0:01:16.61"}]},{"id":10,"text":"for you, or arrange transportation.","confidence":0.8595,"speakerId":6,"language":"en-US","instances":[{"adjustedStart":"0:01:16.61","adjustedEnd":"0:01:18.38","start":"0:01:16.61","end":"0:01:18.38"}]},{"id":11,"text":"Hi this is Mrs Randall.","confidence":0.8456,"speakerId":6,"language":"en-US","instances":[{"adjustedStart":"0:01:18.38","adjustedEnd":"0:01:20.42","start":"0:01:18.38","end":"0:01:20.42"}]},{"id":12,"text":"We just checked out a room 2814.","confidence":0.8456,"speakerId":6,"language":"en-US","instances":[{"adjustedStart":"0:01:20.42","adjustedEnd":"0:01:23.28","start":"0:01:20.42","end":"0:01:23.28"}]},{"id":13,"text":"Nasb Quick Monica please. This is she.","confidence":0.8456,"speakerId":6,"language":"en-US","instances":[{"adjustedStart":"0:01:23.28","adjustedEnd":"0:01:26.13","start":"0:01:23.28","end":"0:01:26.13"}]},{"id":14,"text":"Hi Monica, Hi Hi. We just found out our flight was cancelled.","confidence":0.8456,"speakerId":6,"language":"en-US","instances":[{"adjustedStart":"0:01:26.13","adjustedEnd":"0:01:31.03","start":"0:01:26.13","end":"0:01:31.03"}]},{"id":15,"text":"I know I know. Instead of seeing some horrible airport hotel.","confidence":0.8853,"speakerId":6,"language":"en-US","instances":[{"adjustedStart":"0:01:31.03","adjustedEnd":"0:01:34.9","start":"0:01:31.03","end":"0:01:34.9"}]},{"id":16,"text":"We were hoping that maybe we could get our original room back.","confidence":0.908,"speakerId":6,"language":"en-US","instances":[{"adjustedStart":"0:01:34.9","adjustedEnd":"0:01:38.68","start":"0:01:34.9","end":"0:01:38.68"}]},{"id":17,"text":"Yeah, we can give you that same them back that would be amazing.","confidence":0.908,"speakerId":6,"language":"en-US","instances":[{"adjustedStart":"0:01:38.68","adjustedEnd":"0:01:42.77","start":"0:01:38.68","end":"0:01:42.77"}]},{"id":18,"text":"Thank you so much. Going to grab a bite would it be possible to get the made in there now.","confidence":0.8913,"speakerId":6,"language":"en-US","instances":[{"adjustedStart":"0:01:42.77","adjustedEnd":"0:01:49.05","start":"0:01:42.77","end":"0:01:49.05"}]},{"id":19,"text":"Yes, absolutely perfectly really. Appreciate it,","confidence":0.8871,"speakerId":6,"language":"en-US","instances":[{"adjustedStart":"0:01:49.05","adjustedEnd":"0:01:50.92","start":"0:01:49.05","end":"0:01:50.92"}]},{"id":20,"text":"it's not a problem. Thank you so","confidence":0.8871,"speakerId":6,"language":"en-US","instances":[{"adjustedStart":"0:01:50.92","adjustedEnd":"0:01:53.11","start":"0:01:50.92","end":"0:01:53.11"}]},{"id":21,"text":"much will see you soon.","confidence":0.8871,"speakerId":7,"language":"en-US","instances":[{"adjustedStart":"0:01:53.11","adjustedEnd":"0:01:54.69","start":"0:01:53.11","end":"0:01:54.69"}]},{"id":22,"text":"Hawaii so sorry can","confidence":0.7488,"speakerId":7,"language":"en-US","instances":[{"adjustedStart":"0:01:54.69","adjustedEnd":"0:01:57.38","start":"0:01:54.69","end":"0:01:57.38"}]},{"id":23,"text":"course. Thank you so much have a great day","confidence":0.7488,"speakerId":8,"language":"en-US","instances":[{"adjustedStart":"0:02:05.46","adjustedEnd":"0:02:11.52","start":"0:02:05.46","end":"0:02:11.52"}]},{"id":24,"text":"bye bye.","confidence":0.7488,"speakerId":9,"language":"en-US","instances":[{"adjustedStart":"0:02:11.52","adjustedEnd":"0:02:12.86","start":"0:02:11.52","end":"0:02:12.86"}]}],"ocr":[{"id":1,"text":"FANDANGO","confidence":0.542,"left":116,"top":26,"width":147,"height":18,"language":"en-US","instances":[{"adjustedStart":"0:02:26.355","adjustedEnd":"0:02:27.272","start":"0:02:26.355","end":"0:02:27.272"},{"adjustedStart":"0:02:29.566","adjustedEnd":"0:02:30.484","start":"0:02:29.566","end":"0:02:30.484"},{"adjustedStart":"0:02:31.86","adjustedEnd":"0:02:31.902","start":"0:02:31.86","end":"0:02:31.902"},{"adjustedStart":"0:02:32.319","adjustedEnd":"0:02:32.361","start":"0:02:32.319","end":"0:02:32.361"}]},{"id":2,"text":"WATCH MORE OF","confidence":0.587,"left":778,"top":30,"width":283,"height":24,"language":"en-US","instances":[{"adjustedStart":"0:02:23.602","adjustedEnd":"0:02:31.443","start":"0:02:23.602","end":"0:02:31.443"},{"adjustedStart":"0:02:31.86","adjustedEnd":"0:02:31.902","start":"0:02:31.86","end":"0:02:31.902"},{"adjustedStart":"0:02:32.319","adjustedEnd":"0:02:32.361","start":"0:02:32.319","end":"0:02:32.361"}]},{"id":3,"text":"MOVIECLIPS","confidence":0.3785,"left":116,"top":47,"width":277,"height":34,"language":"en-US","instances":[{"adjustedStart":"0:02:21.308","adjustedEnd":"0:02:21.767","start":"0:02:21.308","end":"0:02:21.767"},{"adjustedStart":"0:02:22.225","adjustedEnd":"0:02:22.684","start":"0:02:22.225","end":"0:02:22.684"},{"adjustedStart":"0:02:23.602","adjustedEnd":"0:02:24.061","start":"0:02:23.602","end":"0:02:24.061"},{"adjustedStart":"0:02:24.519","adjustedEnd":"0:02:24.978","start":"0:02:24.519","end":"0:02:24.978"},{"adjustedStart":"0:02:30.025","adjustedEnd":"0:02:30.484","start":"0:02:30.025","end":"0:02:30.484"},{"adjustedStart":"0:02:31.86","adjustedEnd":"0:02:31.902","start":"0:02:31.86","end":"0:02:31.902"},{"adjustedStart":"0:02:32.319","adjustedEnd":"0:02:32.361","start":"0:02:32.319","end":"0:02:32.361"}]},{"id":4,"text":"•OVIECLIPS","confidence":0.3082,"left":247,"top":47,"width":146,"height":34,"language":"en-US","instances":[{"adjustedStart":"0:02:25.896","adjustedEnd":"0:02:26.814","start":"0:02:25.896","end":"0:02:26.814"},{"adjustedStart":"0:02:27.272","adjustedEnd":"0:02:27.731","start":"0:02:27.272","end":"0:02:27.731"},{"adjustedStart":"0:02:29.566","adjustedEnd":"0:02:30.025","start":"0:02:29.566","end":"0:02:30.025"}]},{"id":5,"text":"YOUR FAVORITE CLIPS","confidence":0.5163,"left":778,"top":60,"width":369,"height":25,"language":"en-US","instances":[{"adjustedStart":"0:02:23.602","adjustedEnd":"0:02:31.443","start":"0:02:23.602","end":"0:02:31.443"},{"adjustedStart":"0:02:31.86","adjustedEnd":"0:02:31.902","start":"0:02:31.86","end":"0:02:31.902"},{"adjustedStart":"0:02:32.319","adjustedEnd":"0:02:32.361","start":"0:02:32.319","end":"0:02:32.361"}]},{"id":6,"text":"SUBSCRIBE","confidence":0.4986,"left":410,"top":188,"width":242,"height":31,"language":"en-US","instances":[{"adjustedStart":"0:02:23.143","adjustedEnd":"0:02:31.443","start":"0:02:23.143","end":"0:02:31.443"},{"adjustedStart":"0:02:31.86","adjustedEnd":"0:02:31.902","start":"0:02:31.86","end":"0:02:31.902"},{"adjustedStart":"0:02:32.319","adjustedEnd":"0:02:32.361","start":"0:02:32.319","end":"0:02:32.361"}]},{"id":7,"text":"ASSO.","confidence":0.3195,"left":791,"top":200,"width":133,"height":34,"language":"de-DE","instances":[{"adjustedStart":"0:01:17.077","adjustedEnd":"0:01:17.536","start":"0:01:17.077","end":"0:01:17.536"},{"adjustedStart":"0:01:18.453","adjustedEnd":"0:01:18.912","start":"0:01:18.453","end":"0:01:18.912"}]},{"id":8,"text":"HERE","confidence":0.3798,"left":542,"top":226,"width":110,"height":30,"language":"en-US","instances":[{"adjustedStart":"0:02:23.143","adjustedEnd":"0:02:31.443","start":"0:02:23.143","end":"0:02:31.443"},{"adjustedStart":"0:02:31.86","adjustedEnd":"0:02:31.902","start":"0:02:31.86","end":"0:02:31.902"},{"adjustedStart":"0:02:32.319","adjustedEnd":"0:02:32.361","start":"0:02:32.319","end":"0:02:32.361"}]},{"id":9,"text":"2814","confidence":0.282,"left":771,"top":246,"width":90,"height":45,"language":"de-DE","instances":[{"adjustedStart":"0:02:06.168","adjustedEnd":"0:02:06.627","start":"0:02:06.168","end":"0:02:06.627"}]},{"id":10,"text":"BUY OR RENT","confidence":0.478,"left":366,"top":497,"width":287,"height":31,"language":"en-US","instances":[{"adjustedStart":"0:02:23.143","adjustedEnd":"0:02:31.443","start":"0:02:23.143","end":"0:02:31.443"},{"adjustedStart":"0:02:31.86","adjustedEnd":"0:02:31.902","start":"0:02:31.86","end":"0:02:31.902"},{"adjustedStart":"0:02:32.319","adjustedEnd":"0:02:32.361","start":"0:02:32.319","end":"0:02:32.361"}]},{"id":11,"text":"THE FULL MOVIE","confidence":0.4402,"left":302,"top":528,"width":350,"height":32,"language":"en-US","instances":[{"adjustedStart":"0:02:23.143","adjustedEnd":"0:02:31.443","start":"0:02:23.143","end":"0:02:31.443"},{"adjustedStart":"0:02:31.86","adjustedEnd":"0:02:31.902","start":"0:02:31.86","end":"0:02:31.902"},{"adjustedStart":"0:02:32.319","adjustedEnd":"0:02:32.361","start":"0:02:32.319","end":"0:02:32.361"}]},{"id":12,"text":"MOVIECLIPS","confidence":0.3416,"left":79,"top":620,"width":173,"height":34,"language":"de-DE","instances":[{"adjustedStart":"0:00:35.327","adjustedEnd":"0:00:38.08","start":"0:00:35.327","end":"0:00:38.08"},{"adjustedStart":"0:00:38.539","adjustedEnd":"0:00:39.456","start":"0:00:38.539","end":"0:00:39.456"},{"adjustedStart":"0:00:39.915","adjustedEnd":"0:00:40.833","start":"0:00:39.915","end":"0:00:40.833"},{"adjustedStart":"0:00:41.291","adjustedEnd":"0:00:42.209","start":"0:00:41.291","end":"0:00:42.209"},{"adjustedStart":"0:00:44.044","adjustedEnd":"0:00:44.503","start":"0:00:44.044","end":"0:00:44.503"},{"adjustedStart":"0:00:44.962","adjustedEnd":"0:00:45.421","start":"0:00:44.962","end":"0:00:45.421"},{"adjustedStart":"0:00:55.514","adjustedEnd":"0:00:55.973","start":"0:00:55.514","end":"0:00:55.973"},{"adjustedStart":"0:00:56.431","adjustedEnd":"0:01:00.102","start":"0:00:56.431","end":"0:01:00.102"},{"adjustedStart":"0:01:01.019","adjustedEnd":"0:01:01.937","start":"0:01:01.019","end":"0:01:01.937"},{"adjustedStart":"0:01:03.772","adjustedEnd":"0:01:04.69","start":"0:01:03.772","end":"0:01:04.69"}]},{"id":13,"text":"FANDANGO","confidence":0.6258,"left":103,"top":650,"width":92,"height":11,"language":"de-DE","instances":[{"adjustedStart":"0:00:01.376","adjustedEnd":"0:00:03.212","start":"0:00:01.376","end":"0:00:03.212"},{"adjustedStart":"0:00:04.588","adjustedEnd":"0:00:05.047","start":"0:00:04.588","end":"0:00:05.047"},{"adjustedStart":"0:00:05.506","adjustedEnd":"0:00:05.965","start":"0:00:05.506","end":"0:00:05.965"},{"adjustedStart":"0:00:06.882","adjustedEnd":"0:00:07.341","start":"0:00:06.882","end":"0:00:07.341"},{"adjustedStart":"0:00:09.176","adjustedEnd":"0:00:11.47","start":"0:00:09.176","end":"0:00:11.47"},{"adjustedStart":"0:00:14.223","adjustedEnd":"0:00:15.14","start":"0:00:14.223","end":"0:00:15.14"},{"adjustedStart":"0:00:16.058","adjustedEnd":"0:00:17.434","start":"0:00:16.058","end":"0:00:17.434"},{"adjustedStart":"0:00:17.893","adjustedEnd":"0:00:18.811","start":"0:00:17.893","end":"0:00:18.811"},{"adjustedStart":"0:00:19.269","adjustedEnd":"0:00:20.646","start":"0:00:19.269","end":"0:00:20.646"},{"adjustedStart":"0:00:21.563","adjustedEnd":"0:00:22.481","start":"0:00:21.563","end":"0:00:22.481"},{"adjustedStart":"0:00:24.316","adjustedEnd":"0:00:25.693","start":"0:00:24.316","end":"0:00:25.693"},{"adjustedStart":"0:00:26.151","adjustedEnd":"0:00:28.445","start":"0:00:26.151","end":"0:00:28.445"},{"adjustedStart":"0:00:29.821","adjustedEnd":"0:00:30.28","start":"0:00:29.821","end":"0:00:30.28"},{"adjustedStart":"0:00:31.198","adjustedEnd":"0:00:31.657","start":"0:00:31.198","end":"0:00:31.657"},{"adjustedStart":"0:00:32.115","adjustedEnd":"0:00:33.951","start":"0:00:32.115","end":"0:00:33.951"},{"adjustedStart":"0:00:34.409","adjustedEnd":"0:00:38.08","start":"0:00:34.409","end":"0:00:38.08"},{"adjustedStart":"0:00:38.539","adjustedEnd":"0:00:39.456","start":"0:00:38.539","end":"0:00:39.456"},{"adjustedStart":"0:00:39.915","adjustedEnd":"0:00:40.833","start":"0:00:39.915","end":"0:00:40.833"},{"adjustedStart":"0:00:41.291","adjustedEnd":"0:00:42.209","start":"0:00:41.291","end":"0:00:42.209"},{"adjustedStart":"0:00:44.044","adjustedEnd":"0:00:44.503","start":"0:00:44.044","end":"0:00:44.503"},{"adjustedStart":"0:00:44.962","adjustedEnd":"0:00:47.715","start":"0:00:44.962","end":"0:00:47.715"},{"adjustedStart":"0:00:49.55","adjustedEnd":"0:00:51.844","start":"0:00:49.55","end":"0:00:51.844"},{"adjustedStart":"0:00:54.137","adjustedEnd":"0:00:54.596","start":"0:00:54.137","end":"0:00:54.596"},{"adjustedStart":"0:00:55.514","adjustedEnd":"0:00:55.973","start":"0:00:55.514","end":"0:00:55.973"},{"adjustedStart":"0:00:56.431","adjustedEnd":"0:01:00.102","start":"0:00:56.431","end":"0:01:00.102"},{"adjustedStart":"0:01:01.019","adjustedEnd":"0:01:01.937","start":"0:01:01.019","end":"0:01:01.937"},{"adjustedStart":"0:01:03.772","adjustedEnd":"0:01:04.69","start":"0:01:03.772","end":"0:01:04.69"},{"adjustedStart":"0:01:05.607","adjustedEnd":"0:01:06.525","start":"0:01:05.607","end":"0:01:06.525"},{"adjustedStart":"0:01:06.984","adjustedEnd":"0:01:09.278","start":"0:01:06.984","end":"0:01:09.278"},{"adjustedStart":"0:01:09.736","adjustedEnd":"0:01:10.654","start":"0:01:09.736","end":"0:01:10.654"},{"adjustedStart":"0:01:11.113","adjustedEnd":"0:01:11.572","start":"0:01:11.113","end":"0:01:11.572"},{"adjustedStart":"0:01:12.03","adjustedEnd":"0:01:16.16","start":"0:01:12.03","end":"0:01:16.16"},{"adjustedStart":"0:01:17.077","adjustedEnd":"0:01:17.536","start":"0:01:17.077","end":"0:01:17.536"},{"adjustedStart":"0:01:17.995","adjustedEnd":"0:01:21.206","start":"0:01:17.995","end":"0:01:21.206"},{"adjustedStart":"0:01:21.665","adjustedEnd":"0:01:23.042","start":"0:01:21.665","end":"0:01:23.042"},{"adjustedStart":"0:01:23.5","adjustedEnd":"0:01:23.959","start":"0:01:23.5","end":"0:01:23.959"},{"adjustedStart":"0:01:24.876","adjustedEnd":"0:01:31.759","start":"0:01:24.876","end":"0:01:31.759"},{"adjustedStart":"0:01:32.217","adjustedEnd":"0:01:40.476","start":"0:01:32.217","end":"0:01:40.476"},{"adjustedStart":"0:01:40.934","adjustedEnd":"0:01:57.91","start":"0:01:40.934","end":"0:01:57.91"},{"adjustedStart":"0:01:58.368","adjustedEnd":"0:02:00.662","start":"0:01:58.368","end":"0:02:00.662"},{"adjustedStart":"0:02:01.121","adjustedEnd":"0:02:07.086","start":"0:02:01.121","end":"0:02:07.086"},{"adjustedStart":"0:02:07.544","adjustedEnd":"0:02:08.921","start":"0:02:07.544","end":"0:02:08.921"},{"adjustedStart":"0:02:09.838","adjustedEnd":"0:02:13.967","start":"0:02:09.838","end":"0:02:13.967"},{"adjustedStart":"0:02:14.426","adjustedEnd":"0:02:20.39","start":"0:02:14.426","end":"0:02:20.39"},{"adjustedStart":"0:02:21.308","adjustedEnd":"0:02:21.767","start":"0:02:21.308","end":"0:02:21.767"},{"adjustedStart":"0:02:22.225","adjustedEnd":"0:02:22.684","start":"0:02:22.225","end":"0:02:22.684"},{"adjustedStart":"0:02:24.519","adjustedEnd":"0:02:24.978","start":"0:02:24.519","end":"0:02:24.978"}]},{"id":14,"text":"MOVIECLIPS","confidence":0.349,"left":95,"top":651,"width":173,"height":25,"language":"de-DE","instances":[{"adjustedStart":"0:00:01.376","adjustedEnd":"0:00:03.212","start":"0:00:01.376","end":"0:00:03.212"},{"adjustedStart":"0:00:04.588","adjustedEnd":"0:00:05.047","start":"0:00:04.588","end":"0:00:05.047"},{"adjustedStart":"0:00:16.058","adjustedEnd":"0:00:17.434","start":"0:00:16.058","end":"0:00:17.434"},{"adjustedStart":"0:00:17.893","adjustedEnd":"0:00:18.811","start":"0:00:17.893","end":"0:00:18.811"},{"adjustedStart":"0:00:19.269","adjustedEnd":"0:00:20.646","start":"0:00:19.269","end":"0:00:20.646"},{"adjustedStart":"0:00:21.563","adjustedEnd":"0:00:22.481","start":"0:00:21.563","end":"0:00:22.481"},{"adjustedStart":"0:00:24.316","adjustedEnd":"0:00:25.693","start":"0:00:24.316","end":"0:00:25.693"},{"adjustedStart":"0:00:26.151","adjustedEnd":"0:00:26.61","start":"0:00:26.151","end":"0:00:26.61"},{"adjustedStart":"0:00:27.069","adjustedEnd":"0:00:28.445","start":"0:00:27.069","end":"0:00:28.445"},{"adjustedStart":"0:00:29.821","adjustedEnd":"0:00:30.28","start":"0:00:29.821","end":"0:00:30.28"},{"adjustedStart":"0:00:45.42","adjustedEnd":"0:00:47.715","start":"0:00:45.42","end":"0:00:47.715"},{"adjustedStart":"0:00:49.55","adjustedEnd":"0:00:51.844","start":"0:00:49.55","end":"0:00:51.844"},{"adjustedStart":"0:00:54.137","adjustedEnd":"0:00:54.596","start":"0:00:54.137","end":"0:00:54.596"},{"adjustedStart":"0:01:05.607","adjustedEnd":"0:01:06.525","start":"0:01:05.607","end":"0:01:06.525"},{"adjustedStart":"0:01:06.984","adjustedEnd":"0:01:09.278","start":"0:01:06.984","end":"0:01:09.278"},{"adjustedStart":"0:01:09.736","adjustedEnd":"0:01:10.654","start":"0:01:09.736","end":"0:01:10.654"},{"adjustedStart":"0:01:11.113","adjustedEnd":"0:01:11.572","start":"0:01:11.113","end":"0:01:11.572"},{"adjustedStart":"0:01:12.03","adjustedEnd":"0:01:15.242","start":"0:01:12.03","end":"0:01:15.242"},{"adjustedStart":"0:01:20.289","adjustedEnd":"0:01:21.206","start":"0:01:20.289","end":"0:01:21.206"},{"adjustedStart":"0:01:21.665","adjustedEnd":"0:01:23.042","start":"0:01:21.665","end":"0:01:23.042"},{"adjustedStart":"0:01:23.5","adjustedEnd":"0:01:23.959","start":"0:01:23.5","end":"0:01:23.959"},{"adjustedStart":"0:01:24.876","adjustedEnd":"0:01:31.759","start":"0:01:24.876","end":"0:01:31.759"},{"adjustedStart":"0:01:32.217","adjustedEnd":"0:01:40.476","start":"0:01:32.217","end":"0:01:40.476"},{"adjustedStart":"0:01:40.934","adjustedEnd":"0:01:45.064","start":"0:01:40.934","end":"0:01:45.064"},{"adjustedStart":"0:01:55.157","adjustedEnd":"0:01:57.91","start":"0:01:55.157","end":"0:01:57.91"},{"adjustedStart":"0:01:58.368","adjustedEnd":"0:02:00.662","start":"0:01:58.368","end":"0:02:00.662"},{"adjustedStart":"0:02:01.121","adjustedEnd":"0:02:07.086","start":"0:02:01.121","end":"0:02:07.086"},{"adjustedStart":"0:02:07.544","adjustedEnd":"0:02:08.921","start":"0:02:07.544","end":"0:02:08.921"},{"adjustedStart":"0:02:09.838","adjustedEnd":"0:02:13.967","start":"0:02:09.838","end":"0:02:13.967"},{"adjustedStart":"0:02:14.426","adjustedEnd":"0:02:20.39","start":"0:02:14.426","end":"0:02:20.39"}]},{"id":15,"text":"MOVIECLIPS","confidence":0.3527,"left":115,"top":677,"width":174,"height":28,"language":"de-DE","instances":[{"adjustedStart":"0:00:05.506","adjustedEnd":"0:00:05.965","start":"0:00:05.506","end":"0:00:05.965"},{"adjustedStart":"0:00:06.882","adjustedEnd":"0:00:07.341","start":"0:00:06.882","end":"0:00:07.341"},{"adjustedStart":"0:00:10.093","adjustedEnd":"0:00:11.47","start":"0:00:10.093","end":"0:00:11.47"},{"adjustedStart":"0:00:14.223","adjustedEnd":"0:00:15.14","start":"0:00:14.223","end":"0:00:15.14"},{"adjustedStart":"0:00:31.198","adjustedEnd":"0:00:31.657","start":"0:00:31.198","end":"0:00:31.657"},{"adjustedStart":"0:00:32.115","adjustedEnd":"0:00:33.033","start":"0:00:32.115","end":"0:00:33.033"},{"adjustedStart":"0:00:33.492","adjustedEnd":"0:00:33.951","start":"0:00:33.492","end":"0:00:33.951"},{"adjustedStart":"0:00:34.409","adjustedEnd":"0:00:35.327","start":"0:00:34.409","end":"0:00:35.327"},{"adjustedStart":"0:01:15.242","adjustedEnd":"0:01:16.16","start":"0:01:15.242","end":"0:01:16.16"},{"adjustedStart":"0:01:17.077","adjustedEnd":"0:01:17.536","start":"0:01:17.077","end":"0:01:17.536"},{"adjustedStart":"0:01:17.995","adjustedEnd":"0:01:20.289","start":"0:01:17.995","end":"0:01:20.289"},{"adjustedStart":"0:01:45.063","adjustedEnd":"0:01:55.157","start":"0:01:45.063","end":"0:01:55.157"}]}],"keywords":[{"id":1,"text":"monica","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:01:10.95","adjustedEnd":"0:01:15.55","start":"0:01:10.95","end":"0:01:15.55"},{"adjustedStart":"0:01:23.28","adjustedEnd":"0:01:26.13","start":"0:01:23.28","end":"0:01:26.13"},{"adjustedStart":"0:01:26.13","adjustedEnd":"0:01:31.03","start":"0:01:26.13","end":"0:01:31.03"}]},{"id":2,"text":"back","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:01:34.9","adjustedEnd":"0:01:38.68","start":"0:01:34.9","end":"0:01:38.68"},{"adjustedStart":"0:01:38.68","adjustedEnd":"0:01:42.77","start":"0:01:38.68","end":"0:01:42.77"}]},{"id":3,"text":"fandango","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:02:26.355","adjustedEnd":"0:02:27.272","start":"0:02:26.355","end":"0:02:27.272"},{"adjustedStart":"0:00:01.376","adjustedEnd":"0:00:03.212","start":"0:00:01.376","end":"0:00:03.212"}]},{"id":4,"text":"movieclips","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:02:21.308","adjustedEnd":"0:02:21.767","start":"0:02:21.308","end":"0:02:21.767"},{"adjustedStart":"0:00:35.327","adjustedEnd":"0:00:38.08","start":"0:00:35.327","end":"0:00:38.08"},{"adjustedStart":"0:00:01.376","adjustedEnd":"0:00:03.212","start":"0:00:01.376","end":"0:00:03.212"},{"adjustedStart":"0:00:05.506","adjustedEnd":"0:00:05.965","start":"0:00:05.506","end":"0:00:05.965"}]}],"topics":[{"id":1,"name":"Public Transportation","referenceId":"Tourism/Public Transportation","referenceType":"VideoIndexer","iptcName":"Lifestyle and Leisure/travel and commuting","confidence":0.3679,"iabName":"Travel","language":"en-US","instances":[{"adjustedStart":"0:00:00","adjustedEnd":"0:02:32.462","start":"0:00:00","end":"0:02:32.462"}]},{"id":2,"name":"Flights","referenceId":"Tourism/Visiting and Travel/Flights","referenceType":"VideoIndexer","iptcName":"Lifestyle and Leisure/travel and commuting","confidence":0.2591,"iabName":"Travel","language":"en-US","instances":[{"adjustedStart":"0:00:00","adjustedEnd":"0:02:32.462","start":"0:00:00","end":"0:02:32.462"}]},{"id":3,"name":"Aviation","referenceId":"Aviation","referenceType":"VideoIndexer","confidence":0.2074,"iabName":null,"language":"en-US","instances":[{"adjustedStart":"0:00:00","adjustedEnd":"0:02:32.462","start":"0:00:00","end":"0:02:32.462"}]}],"faces":[{"id":1003,"name":"Unknown #1","confidence":0,"description":null,"thumbnailId":"6dfe2efd-4caf-48ce-b144-10f6a4c10f7e","title":null,"imageUrl":null,"thumbnails":[{"id":"7f659abb-1a51-44a9-9c3f-26537f5c86fc","fileName":"FaceInstanceThumbnail_7f659abb-1a51-44a9-9c3f-26537f5c86fc.jpg","instances":[{"adjustedStart":"0:00:05.589","adjustedEnd":"0:00:05.631","start":"0:00:05.589","end":"0:00:05.631"}]},{"id":"6dfe2efd-4caf-48ce-b144-10f6a4c10f7e","fileName":"FaceInstanceThumbnail_6dfe2efd-4caf-48ce-b144-10f6a4c10f7e.jpg","instances":[{"adjustedStart":"0:00:07.674","adjustedEnd":"0:00:07.716","start":"0:00:07.674","end":"0:00:07.716"}]},{"id":"a4e66c67-e8af-4991-896c-45216e3d74e8","fileName":"FaceInstanceThumbnail_a4e66c67-e8af-4991-896c-45216e3d74e8.jpg","instances":[{"adjustedStart":"0:00:15.015","adjustedEnd":"0:00:15.057","start":"0:00:15.015","end":"0:00:15.057"}]},{"id":"8c02fd80-7221-40b9-ac7f-d83dbc72fe2e","fileName":"FaceInstanceThumbnail_8c02fd80-7221-40b9-ac7f-d83dbc72fe2e.jpg","instances":[{"adjustedStart":"0:00:20.437","adjustedEnd":"0:00:20.479","start":"0:00:20.437","end":"0:00:20.479"}]},{"id":"d2e51d7a-3999-4f0e-b9f3-5cc8b30771a9","fileName":"FaceInstanceThumbnail_d2e51d7a-3999-4f0e-b9f3-5cc8b30771a9.jpg","instances":[{"adjustedStart":"0:00:32.616","adjustedEnd":"0:00:32.658","start":"0:00:32.616","end":"0:00:32.658"}]},{"id":"9790a07b-3b0b-4f15-958b-7195b6142a70","fileName":"FaceInstanceThumbnail_9790a07b-3b0b-4f15-958b-7195b6142a70.jpg","instances":[{"adjustedStart":"0:00:37.621","adjustedEnd":"0:00:37.663","start":"0:00:37.621","end":"0:00:37.663"}]},{"id":"49e9f0c0-f3e1-4756-8eae-00d15c9fdbdd","fileName":"FaceInstanceThumbnail_49e9f0c0-f3e1-4756-8eae-00d15c9fdbdd.jpg","instances":[{"adjustedStart":"0:00:42.793","adjustedEnd":"0:00:42.835","start":"0:00:42.793","end":"0:00:42.835"}]},{"id":"bbdd5a09-2d7d-4676-9752-8af2d48a023b","fileName":"FaceInstanceThumbnail_bbdd5a09-2d7d-4676-9752-8af2d48a023b.jpg","instances":[{"adjustedStart":"0:00:53.971","adjustedEnd":"0:00:54.013","start":"0:00:53.971","end":"0:00:54.013"}]},{"id":"8f383043-90fe-40e9-8a8b-e11be3e20bce","fileName":"FaceInstanceThumbnail_8f383043-90fe-40e9-8a8b-e11be3e20bce.jpg","instances":[{"adjustedStart":"0:01:24.585","adjustedEnd":"0:01:24.627","start":"0:01:24.585","end":"0:01:24.627"}]},{"id":"07086a1d-e536-4e9e-8ab5-e194121b305b","fileName":"FaceInstanceThumbnail_07086a1d-e536-4e9e-8ab5-e194121b305b.jpg","instances":[{"adjustedStart":"0:01:29.59","adjustedEnd":"0:01:29.632","start":"0:01:29.59","end":"0:01:29.632"}]},{"id":"c462ebf2-2945-4a34-99d5-b4b2c2731f10","fileName":"FaceInstanceThumbnail_c462ebf2-2945-4a34-99d5-b4b2c2731f10.jpg","instances":[{"adjustedStart":"0:01:34.595","adjustedEnd":"0:01:34.637","start":"0:01:34.595","end":"0:01:34.637"}]},{"id":"e8982b94-22fc-4e25-8a8e-52189c467e0a","fileName":"FaceInstanceThumbnail_e8982b94-22fc-4e25-8a8e-52189c467e0a.jpg","instances":[{"adjustedStart":"0:01:39.6","adjustedEnd":"0:01:39.642","start":"0:01:39.6","end":"0:01:39.642"}]},{"id":"6d1d8f3c-4db4-471a-9a31-2ef26457ed20","fileName":"FaceInstanceThumbnail_6d1d8f3c-4db4-471a-9a31-2ef26457ed20.jpg","instances":[{"adjustedStart":"0:01:50.611","adjustedEnd":"0:01:50.653","start":"0:01:50.611","end":"0:01:50.653"}]},{"id":"d3126bd0-5c3b-47d9-85c9-969684c102e3","fileName":"FaceInstanceThumbnail_d3126bd0-5c3b-47d9-85c9-969684c102e3.jpg","instances":[{"adjustedStart":"0:01:55.782","adjustedEnd":"0:01:55.824","start":"0:01:55.782","end":"0:01:55.824"}]},{"id":"948f1cfc-6a4b-4401-8bef-effe84f36e84","fileName":"FaceInstanceThumbnail_948f1cfc-6a4b-4401-8bef-effe84f36e84.jpg","instances":[{"adjustedStart":"0:02:01.038","adjustedEnd":"0:02:01.08","start":"0:02:01.038","end":"0:02:01.08"}]},{"id":"52ca2c66-b902-4d02-a273-1bc7eec4b993","fileName":"FaceInstanceThumbnail_52ca2c66-b902-4d02-a273-1bc7eec4b993.jpg","instances":[{"adjustedStart":"0:02:14.551","adjustedEnd":"0:02:14.593","start":"0:02:14.551","end":"0:02:14.593"}]}],"instances":[{"thumbnailsIds":["7f659abb-1a51-44a9-9c3f-26537f5c86fc","6dfe2efd-4caf-48ce-b144-10f6a4c10f7e"],"adjustedStart":"0:00:05.589","adjustedEnd":"0:00:09.927","start":"0:00:05.589","end":"0:00:09.927"},{"thumbnailsIds":["a4e66c67-e8af-4991-896c-45216e3d74e8"],"adjustedStart":"0:00:15.015","adjustedEnd":"0:00:18.852","start":"0:00:15.015","end":"0:00:18.852"},{"thumbnailsIds":["8c02fd80-7221-40b9-ac7f-d83dbc72fe2e"],"adjustedStart":"0:00:20.437","adjustedEnd":"0:00:21.564","start":"0:00:20.437","end":"0:00:21.564"},{"adjustedStart":"0:00:22.189","adjustedEnd":"0:00:22.273","start":"0:00:22.189","end":"0:00:22.273"},{"adjustedStart":"0:00:24.191","adjustedEnd":"0:00:24.9","start":"0:00:24.191","end":"0:00:24.9"},{"adjustedStart":"0:00:26.693","adjustedEnd":"0:00:27.152","start":"0:00:26.693","end":"0:00:27.152"},{"thumbnailsIds":["d2e51d7a-3999-4f0e-b9f3-5cc8b30771a9"],"adjustedStart":"0:00:29.613","adjustedEnd":"0:00:34.702","start":"0:00:29.613","end":"0:00:34.702"},{"thumbnailsIds":["9790a07b-3b0b-4f15-958b-7195b6142a70"],"adjustedStart":"0:00:36.119","adjustedEnd":"0:00:38.831","start":"0:00:36.119","end":"0:00:38.831"},{"thumbnailsIds":["49e9f0c0-f3e1-4756-8eae-00d15c9fdbdd"],"adjustedStart":"0:00:40.707","adjustedEnd":"0:00:43.335","start":"0:00:40.707","end":"0:00:43.335"},{"adjustedStart":"0:00:45.045","adjustedEnd":"0:00:47.423","start":"0:00:45.045","end":"0:00:47.423"},{"thumbnailsIds":["bbdd5a09-2d7d-4676-9752-8af2d48a023b"],"adjustedStart":"0:00:49.049","adjustedEnd":"0:00:56.181","start":"0:00:49.049","end":"0:00:56.181"},{"thumbnailsIds":["8f383043-90fe-40e9-8a8b-e11be3e20bce","07086a1d-e536-4e9e-8ab5-e194121b305b","c462ebf2-2945-4a34-99d5-b4b2c2731f10","e8982b94-22fc-4e25-8a8e-52189c467e0a"],"adjustedStart":"0:01:24.585","adjustedEnd":"0:01:40.934","start":"0:01:24.585","end":"0:01:40.934"},{"thumbnailsIds":["6d1d8f3c-4db4-471a-9a31-2ef26457ed20"],"adjustedStart":"0:01:50.611","adjustedEnd":"0:01:54.323","start":"0:01:50.611","end":"0:01:54.323"},{"thumbnailsIds":["d3126bd0-5c3b-47d9-85c9-969684c102e3","948f1cfc-6a4b-4401-8bef-effe84f36e84"],"adjustedStart":"0:01:55.782","adjustedEnd":"0:02:05.209","start":"0:01:55.782","end":"0:02:05.209"},{"thumbnailsIds":["52ca2c66-b902-4d02-a273-1bc7eec4b993"],"adjustedStart":"0:02:14.551","adjustedEnd":"0:02:16.011","start":"0:02:14.551","end":"0:02:16.011"},{"adjustedStart":"0:02:16.553","adjustedEnd":"0:02:18.847","start":"0:02:16.553","end":"0:02:18.847"}]},{"id":1336,"name":"Unknown #2","confidence":0,"description":null,"thumbnailId":"89853eee-7f72-451c-96fb-3a38cc8f3a34","title":null,"imageUrl":null,"thumbnails":[{"id":"f8f66e79-993c-4978-beea-31f800c947e3","fileName":"FaceInstanceThumbnail_f8f66e79-993c-4978-beea-31f800c947e3.jpg","instances":[{"adjustedStart":"0:00:47.798","adjustedEnd":"0:00:47.84","start":"0:00:47.798","end":"0:00:47.84"}]},{"id":"89853eee-7f72-451c-96fb-3a38cc8f3a34","fileName":"FaceInstanceThumbnail_89853eee-7f72-451c-96fb-3a38cc8f3a34.jpg","instances":[{"adjustedStart":"0:00:48.799","adjustedEnd":"0:00:48.841","start":"0:00:48.799","end":"0:00:48.841"}]}],"instances":[{"adjustedStart":"0:00:34.701","adjustedEnd":"0:00:36.12","start":"0:00:34.701","end":"0:00:36.12"},{"adjustedStart":"0:00:38.872","adjustedEnd":"0:00:42.751","start":"0:00:38.872","end":"0:00:42.751"},{"adjustedStart":"0:00:43.377","adjustedEnd":"0:00:45.045","start":"0:00:43.377","end":"0:00:45.045"},{"thumbnailsIds":["f8f66e79-993c-4978-beea-31f800c947e3","89853eee-7f72-451c-96fb-3a38cc8f3a34"],"adjustedStart":"0:00:47.464","adjustedEnd":"0:00:48.924","start":"0:00:47.464","end":"0:00:48.924"}]},{"id":1586,"name":"Unknown #4","confidence":0,"description":null,"thumbnailId":"6e01e0bf-abf4-4393-adb5-a9d7cf9aae4a","title":null,"imageUrl":null,"thumbnails":[{"id":"6e01e0bf-abf4-4393-adb5-a9d7cf9aae4a","fileName":"FaceInstanceThumbnail_6e01e0bf-abf4-4393-adb5-a9d7cf9aae4a.jpg","instances":[{"adjustedStart":"0:01:51.028","adjustedEnd":"0:01:51.07","start":"0:01:51.028","end":"0:01:51.07"}]},{"id":"3c76e4b8-0454-467d-b734-4cf773adf1fc","fileName":"FaceInstanceThumbnail_3c76e4b8-0454-467d-b734-4cf773adf1fc.jpg","instances":[{"adjustedStart":"0:02:20.891","adjustedEnd":"0:02:20.933","start":"0:02:20.891","end":"0:02:20.933"}]},{"id":"9e446f18-b60f-4303-a4fa-3406cbb97319","fileName":"FaceInstanceThumbnail_9e446f18-b60f-4303-a4fa-3406cbb97319.jpg","instances":[{"adjustedStart":"0:02:21.808","adjustedEnd":"0:02:21.85","start":"0:02:21.808","end":"0:02:21.85"}]}],"instances":[{"thumbnailsIds":["6e01e0bf-abf4-4393-adb5-a9d7cf9aae4a"],"adjustedStart":"0:01:51.028","adjustedEnd":"0:01:52.78","start":"0:01:51.028","end":"0:01:52.78"},{"thumbnailsIds":["3c76e4b8-0454-467d-b734-4cf773adf1fc","9e446f18-b60f-4303-a4fa-3406cbb97319"],"adjustedStart":"0:02:20.891","adjustedEnd":"0:02:22.434","start":"0:02:20.891","end":"0:02:22.434"}]},{"id":1083,"name":"Unknown #3","confidence":0,"description":null,"thumbnailId":"73964295-dfd9-4fea-8540-853126bb64f9","title":null,"imageUrl":null,"thumbnails":[{"id":"73964295-dfd9-4fea-8540-853126bb64f9","fileName":"FaceInstanceThumbnail_73964295-dfd9-4fea-8540-853126bb64f9.jpg","instances":[{"adjustedStart":"0:00:24.942","adjustedEnd":"0:00:24.984","start":"0:00:24.942","end":"0:00:24.984"}]}],"instances":[{"thumbnailsIds":["73964295-dfd9-4fea-8540-853126bb64f9"],"adjustedStart":"0:00:24.942","adjustedEnd":"0:00:27.152","start":"0:00:24.942","end":"0:00:27.152"}]},{"id":1095,"name":"Unknown #6","confidence":0,"description":null,"thumbnailId":"c9c8008b-80e1-44ff-a8c6-27b44b141d16","title":null,"imageUrl":null,"thumbnails":[{"id":"ecb901fd-5a13-43cc-9301-cc5efc8bdfe5","fileName":"FaceInstanceThumbnail_ecb901fd-5a13-43cc-9301-cc5efc8bdfe5.jpg","instances":[{"adjustedStart":"0:00:27.194","adjustedEnd":"0:00:27.236","start":"0:00:27.194","end":"0:00:27.236"}]},{"id":"c9c8008b-80e1-44ff-a8c6-27b44b141d16","fileName":"FaceInstanceThumbnail_c9c8008b-80e1-44ff-a8c6-27b44b141d16.jpg","instances":[{"adjustedStart":"0:00:28.362","adjustedEnd":"0:00:28.404","start":"0:00:28.362","end":"0:00:28.404"}]}],"instances":[{"thumbnailsIds":["ecb901fd-5a13-43cc-9301-cc5efc8bdfe5","c9c8008b-80e1-44ff-a8c6-27b44b141d16"],"adjustedStart":"0:00:27.194","adjustedEnd":"0:00:28.404","start":"0:00:27.194","end":"0:00:28.404"}]}],"labels":[{"id":1,"name":"indoor","language":"en-US","instances":[{"confidence":0.9681,"adjustedStart":"0:00:02.669","adjustedEnd":"0:00:57.391","start":"0:00:02.669","end":"0:00:57.391"},{"confidence":0.9566,"adjustedStart":"0:01:00.06","adjustedEnd":"0:01:16.076","start":"0:01:00.06","end":"0:01:16.076"},{"confidence":0.8562,"adjustedStart":"0:01:17.411","adjustedEnd":"0:01:18.746","start":"0:01:17.411","end":"0:01:18.746"},{"confidence":0.8673,"adjustedStart":"0:01:24.084","adjustedEnd":"0:01:25.419","start":"0:01:24.084","end":"0:01:25.419"},{"confidence":0.9533,"adjustedStart":"0:01:26.753","adjustedEnd":"0:01:49.443","start":"0:01:26.753","end":"0:01:49.443"},{"confidence":0.9624,"adjustedStart":"0:01:50.777","adjustedEnd":"0:01:53.447","start":"0:01:50.777","end":"0:01:53.447"},{"confidence":0.9747,"adjustedStart":"0:01:54.781","adjustedEnd":"0:02:04.124","start":"0:01:54.781","end":"0:02:04.124"},{"confidence":0.9439,"adjustedStart":"0:02:05.459","adjustedEnd":"0:02:22.81","start":"0:02:05.459","end":"0:02:22.81"}]},{"id":2,"name":"person","referenceId":"person","language":"en-US","instances":[{"confidence":0.9737,"adjustedStart":"0:00:05.339","adjustedEnd":"0:00:12.012","start":"0:00:05.339","end":"0:00:12.012"},{"confidence":0.9719,"adjustedStart":"0:00:14.681","adjustedEnd":"0:00:21.355","start":"0:00:14.681","end":"0:00:21.355"},{"confidence":0.987,"adjustedStart":"0:00:22.689","adjustedEnd":"0:00:57.391","start":"0:00:22.689","end":"0:00:57.391"},{"confidence":0.9168,"adjustedStart":"0:01:01.395","adjustedEnd":"0:01:04.064","start":"0:01:01.395","end":"0:01:04.064"},{"confidence":0.9716,"adjustedStart":"0:01:05.399","adjustedEnd":"0:01:16.076","start":"0:01:05.399","end":"0:01:16.076"},{"confidence":0.9784,"adjustedStart":"0:01:25.419","adjustedEnd":"0:01:41.435","start":"0:01:25.419","end":"0:01:41.435"},{"confidence":0.9155,"adjustedStart":"0:01:45.439","adjustedEnd":"0:01:49.443","start":"0:01:45.439","end":"0:01:49.443"},{"confidence":0.9763,"adjustedStart":"0:01:50.777","adjustedEnd":"0:01:53.447","start":"0:01:50.777","end":"0:01:53.447"},{"confidence":0.9457,"adjustedStart":"0:01:54.781","adjustedEnd":"0:01:58.786","start":"0:01:54.781","end":"0:01:58.786"},{"confidence":0.9923,"adjustedStart":"0:02:00.12","adjustedEnd":"0:02:04.124","start":"0:02:00.12","end":"0:02:04.124"},{"confidence":0.9781,"adjustedStart":"0:02:05.459","adjustedEnd":"0:02:12.132","start":"0:02:05.459","end":"0:02:12.132"},{"confidence":0.9528,"adjustedStart":"0:02:13.467","adjustedEnd":"0:02:14.802","start":"0:02:13.467","end":"0:02:14.802"},{"confidence":0.9471,"adjustedStart":"0:02:17.471","adjustedEnd":"0:02:20.14","start":"0:02:17.471","end":"0:02:20.14"}]},{"id":3,"name":"woman","referenceId":"person/woman","language":"en-US","instances":[{"confidence":0.9167,"adjustedStart":"0:00:08.008","adjustedEnd":"0:00:10.678","start":"0:00:08.008","end":"0:00:10.678"},{"confidence":0.9054,"adjustedStart":"0:00:28.028","adjustedEnd":"0:00:29.363","start":"0:00:28.028","end":"0:00:29.363"},{"confidence":0.9427,"adjustedStart":"0:00:33.367","adjustedEnd":"0:00:48.048","start":"0:00:33.367","end":"0:00:48.048"},{"confidence":0.9479,"adjustedStart":"0:00:49.383","adjustedEnd":"0:00:50.718","start":"0:00:49.383","end":"0:00:50.718"},{"confidence":0.9096,"adjustedStart":"0:00:53.387","adjustedEnd":"0:00:54.722","start":"0:00:53.387","end":"0:00:54.722"},{"confidence":0.9482,"adjustedStart":"0:02:01.455","adjustedEnd":"0:02:04.124","start":"0:02:01.455","end":"0:02:04.124"}]},{"id":4,"name":"wall","referenceId":"structure/wall","language":"en-US","instances":[{"confidence":0.972,"adjustedStart":"0:00:18.685","adjustedEnd":"0:00:20.02","start":"0:00:18.685","end":"0:00:20.02"},{"confidence":0.9625,"adjustedStart":"0:00:28.028","adjustedEnd":"0:00:30.698","start":"0:00:28.028","end":"0:00:30.698"},{"confidence":0.9765,"adjustedStart":"0:00:32.032","adjustedEnd":"0:00:34.702","start":"0:00:32.032","end":"0:00:34.702"},{"confidence":0.951,"adjustedStart":"0:00:37.371","adjustedEnd":"0:00:38.706","start":"0:00:37.371","end":"0:00:38.706"},{"confidence":0.9616,"adjustedStart":"0:00:45.379","adjustedEnd":"0:00:46.714","start":"0:00:45.379","end":"0:00:46.714"},{"confidence":0.9548,"adjustedStart":"0:00:49.383","adjustedEnd":"0:00:50.718","start":"0:00:49.383","end":"0:00:50.718"},{"confidence":0.9545,"adjustedStart":"0:00:52.052","adjustedEnd":"0:00:53.387","start":"0:00:52.052","end":"0:00:53.387"},{"confidence":0.9664,"adjustedStart":"0:01:08.068","adjustedEnd":"0:01:09.403","start":"0:01:08.068","end":"0:01:09.403"},{"confidence":0.9825,"adjustedStart":"0:01:41.435","adjustedEnd":"0:01:45.439","start":"0:01:41.435","end":"0:01:45.439"},{"confidence":0.9773,"adjustedStart":"0:01:54.781","adjustedEnd":"0:01:58.786","start":"0:01:54.781","end":"0:01:58.786"},{"confidence":0.9891,"adjustedStart":"0:02:00.12","adjustedEnd":"0:02:04.124","start":"0:02:00.12","end":"0:02:04.124"},{"confidence":0.97,"adjustedStart":"0:02:06.793","adjustedEnd":"0:02:10.798","start":"0:02:06.793","end":"0:02:10.798"},{"confidence":0.9729,"adjustedStart":"0:02:12.132","adjustedEnd":"0:02:22.81","start":"0:02:12.132","end":"0:02:22.81"}]},{"id":5,"name":"mirror","referenceId":"mirror","language":"en-US","instances":[{"confidence":0.9362,"adjustedStart":"0:00:37.371","adjustedEnd":"0:00:40.04","start":"0:00:37.371","end":"0:00:40.04"},{"confidence":0.8904,"adjustedStart":"0:00:46.713","adjustedEnd":"0:00:48.048","start":"0:00:46.713","end":"0:00:48.048"}]},{"id":6,"name":"looking","referenceId":"look/looking","language":"en-US","instances":[{"confidence":0.8927,"adjustedStart":"0:00:37.371","adjustedEnd":"0:00:40.04","start":"0:00:37.371","end":"0:00:40.04"},{"confidence":0.8567,"adjustedStart":"0:00:41.375","adjustedEnd":"0:00:44.044","start":"0:00:41.375","end":"0:00:44.044"},{"confidence":0.8334,"adjustedStart":"0:00:45.379","adjustedEnd":"0:00:46.714","start":"0:00:45.379","end":"0:00:46.714"},{"confidence":0.8468,"adjustedStart":"0:01:45.439","adjustedEnd":"0:01:46.774","start":"0:01:45.439","end":"0:01:46.774"},{"confidence":0.8302,"adjustedStart":"0:02:02.789","adjustedEnd":"0:02:04.124","start":"0:02:02.789","end":"0:02:04.124"}]},{"id":7,"name":"standing","language":"en-US","instances":[{"confidence":0.8096,"adjustedStart":"0:00:50.717","adjustedEnd":"0:00:52.052","start":"0:00:50.717","end":"0:00:52.052"},{"confidence":0.8101,"adjustedStart":"0:01:09.403","adjustedEnd":"0:01:10.738","start":"0:01:09.403","end":"0:01:10.738"}]},{"id":8,"name":"laptop","referenceId":"computer/portable computer/laptop","language":"en-US","instances":[{"confidence":0.9377,"adjustedStart":"0:01:14.741","adjustedEnd":"0:01:16.076","start":"0:01:14.741","end":"0:01:16.076"}]},{"id":9,"name":"man","referenceId":"person/man","language":"en-US","instances":[{"confidence":0.9226,"adjustedStart":"0:01:22.749","adjustedEnd":"0:01:24.084","start":"0:01:22.749","end":"0:01:24.084"},{"confidence":0.9703,"adjustedStart":"0:02:06.793","adjustedEnd":"0:02:12.132","start":"0:02:06.793","end":"0:02:12.132"}]},{"id":10,"name":"cellphone","referenceId":"electronic equipment/telephone/cellular telephone/cellphone","language":"en-US","instances":[{"confidence":0.8381,"adjustedStart":"0:01:25.419","adjustedEnd":"0:01:26.754","start":"0:01:25.419","end":"0:01:26.754"}]},{"id":11,"name":"cat","referenceId":"animal/cat","language":"en-US","instances":[{"confidence":0.9565,"adjustedStart":"0:02:16.136","adjustedEnd":"0:02:17.471","start":"0:02:16.136","end":"0:02:17.471"}]},{"id":12,"name":"bed","referenceId":"furniture/bed","language":"en-US","instances":[{"confidence":0.9397,"adjustedStart":"0:02:21.475","adjustedEnd":"0:02:22.81","start":"0:02:21.475","end":"0:02:22.81"}]}],"scenes":[{"id":1,"instances":[{"adjustedStart":"0:00:00","adjustedEnd":"0:00:47.464","start":"0:00:00","end":"0:00:47.464"}]},{"id":2,"instances":[{"adjustedStart":"0:00:47.464","adjustedEnd":"0:00:49.299","start":"0:00:47.464","end":"0:00:49.299"}]},{"id":3,"instances":[{"adjustedStart":"0:00:49.299","adjustedEnd":"0:00:57.933","start":"0:00:49.299","end":"0:00:57.933"}]},{"id":4,"instances":[{"adjustedStart":"0:00:57.933","adjustedEnd":"0:01:19.454","start":"0:00:57.933","end":"0:01:19.454"}]},{"id":5,"instances":[{"adjustedStart":"0:01:19.454","adjustedEnd":"0:01:24.626","start":"0:01:19.454","end":"0:01:24.626"}]},{"id":6,"instances":[{"adjustedStart":"0:01:24.626","adjustedEnd":"0:01:42.644","start":"0:01:24.626","end":"0:01:42.644"}]},{"id":7,"instances":[{"adjustedStart":"0:01:42.644","adjustedEnd":"0:02:06.376","start":"0:01:42.644","end":"0:02:06.376"}]},{"id":8,"instances":[{"adjustedStart":"0:02:06.376","adjustedEnd":"0:02:22.726","start":"0:02:06.376","end":"0:02:22.726"}]},{"id":9,"instances":[{"adjustedStart":"0:02:22.726","adjustedEnd":"0:02:24.227","start":"0:02:22.726","end":"0:02:24.227"}]},{"id":10,"instances":[{"adjustedStart":"0:02:24.227","adjustedEnd":"0:02:28.69","start":"0:02:24.227","end":"0:02:28.69"}]},{"id":11,"instances":[{"adjustedStart":"0:02:28.69","adjustedEnd":"0:02:30.776","start":"0:02:28.69","end":"0:02:30.776"}]},{"id":12,"instances":[{"adjustedStart":"0:02:30.776","adjustedEnd":"0:02:32.444","start":"0:02:30.776","end":"0:02:32.444"}]}],"shots":[{"id":1,"tags":["Indoor","Medium","CenterFace"],"keyFrames":[{"id":1,"instances":[{"thumbnailId":"8a3ebbb3-bd75-4a72-b3cd-4071c94b9bdd","adjustedStart":"0:00:02.878","adjustedEnd":"0:00:02.92","start":"0:00:02.878","end":"0:00:02.92"}]},{"id":2,"instances":[{"thumbnailId":"1b5209b0-7c9b-4da8-be9c-91f9943b6001","adjustedStart":"0:00:09.468","adjustedEnd":"0:00:09.51","start":"0:00:09.468","end":"0:00:09.51"}]},{"id":3,"instances":[{"thumbnailId":"fd6a994c-353d-47d2-aa00-c70041edf892","adjustedStart":"0:00:14.473","adjustedEnd":"0:00:14.515","start":"0:00:14.473","end":"0:00:14.515"}]}],"instances":[{"adjustedStart":"0:00:00","adjustedEnd":"0:00:19.144","start":"0:00:00","end":"0:00:19.144"}]},{"id":2,"tags":["Indoor"],"keyFrames":[{"id":4,"instances":[{"thumbnailId":"2731dc2a-a027-41d5-92e4-cccbf45dbdfe","adjustedStart":"0:00:19.144","adjustedEnd":"0:00:19.186","start":"0:00:19.144","end":"0:00:19.186"}]},{"id":5,"instances":[{"thumbnailId":"da79cd7d-52c7-4f86-ba48-4a5e7ac83736","adjustedStart":"0:00:19.895","adjustedEnd":"0:00:19.937","start":"0:00:19.895","end":"0:00:19.937"}]},{"id":6,"instances":[{"thumbnailId":"9c2954b2-dfc3-4b8d-b3e9-2269cd50a3c2","adjustedStart":"0:00:21.605","adjustedEnd":"0:00:21.647","start":"0:00:21.605","end":"0:00:21.647"}]}],"instances":[{"adjustedStart":"0:00:19.144","adjustedEnd":"0:00:21.855","start":"0:00:19.144","end":"0:00:21.855"}]},{"id":3,"tags":["Indoor"],"keyFrames":[{"id":7,"instances":[{"thumbnailId":"ae99ef9a-c3ff-4a6f-991b-53b448a3a28b","adjustedStart":"0:00:21.855","adjustedEnd":"0:00:21.897","start":"0:00:21.855","end":"0:00:21.897"}]},{"id":8,"instances":[{"thumbnailId":"b93571e3-b14d-4a1d-b328-2b6fedd925df","adjustedStart":"0:00:23.941","adjustedEnd":"0:00:23.983","start":"0:00:23.941","end":"0:00:23.983"}]}],"instances":[{"adjustedStart":"0:00:21.855","adjustedEnd":"0:00:27.194","start":"0:00:21.855","end":"0:00:27.194"}]},{"id":4,"tags":["Medium","RightFace","Indoor","Wide","CenterFace"],"keyFrames":[{"id":9,"instances":[{"thumbnailId":"19853666-a7f6-4bd5-b30c-f9cad2fe4c2b","adjustedStart":"0:00:27.194","adjustedEnd":"0:00:27.236","start":"0:00:27.194","end":"0:00:27.236"}]},{"id":10,"instances":[{"thumbnailId":"f2d1852f-ab90-47d3-bf3f-89ace030dec2","adjustedStart":"0:00:30.239","adjustedEnd":"0:00:30.281","start":"0:00:30.239","end":"0:00:30.281"}]}],"instances":[{"adjustedStart":"0:00:27.194","adjustedEnd":"0:00:34.743","start":"0:00:27.194","end":"0:00:34.743"}]},{"id":5,"tags":["Medium","RightFace","Indoor"],"keyFrames":[{"id":11,"instances":[{"thumbnailId":"a0eb54c7-4476-49c7-9c84-92be0b16b779","adjustedStart":"0:00:34.743","adjustedEnd":"0:00:34.785","start":"0:00:34.743","end":"0:00:34.785"}]}],"instances":[{"adjustedStart":"0:00:34.743","adjustedEnd":"0:00:36.411","start":"0:00:34.743","end":"0:00:36.411"}]},{"id":6,"tags":["Medium","LeftFace","Indoor"],"keyFrames":[{"id":12,"instances":[{"thumbnailId":"f0777921-6124-4479-9fc5-26ffc16d5ed1","adjustedStart":"0:00:36.411","adjustedEnd":"0:00:36.453","start":"0:00:36.411","end":"0:00:36.453"}]}],"instances":[{"adjustedStart":"0:00:36.411","adjustedEnd":"0:00:38.872","start":"0:00:36.411","end":"0:00:38.872"}]},{"id":7,"tags":["Medium","CenterFace","Indoor"],"keyFrames":[{"id":13,"instances":[{"thumbnailId":"8c630090-6dfb-460b-8813-7f99bb94b1bb","adjustedStart":"0:00:38.872","adjustedEnd":"0:00:38.914","start":"0:00:38.872","end":"0:00:38.914"}]}],"instances":[{"adjustedStart":"0:00:38.872","adjustedEnd":"0:00:40.999","start":"0:00:38.872","end":"0:00:40.999"}]},{"id":8,"tags":["Medium","Indoor"],"keyFrames":[{"id":14,"instances":[{"thumbnailId":"0b8bfbb1-6f42-4752-93ed-1f4476ce36aa","adjustedStart":"0:00:40.999","adjustedEnd":"0:00:41.041","start":"0:00:40.999","end":"0:00:41.041"}]}],"instances":[{"adjustedStart":"0:00:40.999","adjustedEnd":"0:00:43.377","start":"0:00:40.999","end":"0:00:43.377"}]},{"id":9,"tags":["Medium","RightFace","Indoor"],"keyFrames":[{"id":15,"instances":[{"thumbnailId":"94656917-9f3a-4154-97fd-bcad9de623e5","adjustedStart":"0:00:43.377","adjustedEnd":"0:00:43.419","start":"0:00:43.377","end":"0:00:43.419"}]}],"instances":[{"adjustedStart":"0:00:43.377","adjustedEnd":"0:00:45.337","start":"0:00:43.377","end":"0:00:45.337"}]},{"id":10,"tags":["Medium","LeftFace","Indoor"],"keyFrames":[{"id":16,"instances":[{"thumbnailId":"3f5bd8ef-492c-4073-9da8-a82630d55b8e","adjustedStart":"0:00:45.337","adjustedEnd":"0:00:45.379","start":"0:00:45.337","end":"0:00:45.379"}]}],"instances":[{"adjustedStart":"0:00:45.337","adjustedEnd":"0:00:47.464","start":"0:00:45.337","end":"0:00:47.464"}]},{"id":11,"tags":["Medium","RightFace","Indoor"],"keyFrames":[{"id":17,"instances":[{"thumbnailId":"26a69e65-2a62-41de-9f21-a1cce529ceaf","adjustedStart":"0:00:47.464","adjustedEnd":"0:00:47.506","start":"0:00:47.464","end":"0:00:47.506"}]},{"id":18,"instances":[{"thumbnailId":"ae055810-0c5f-4acd-8d59-781420dbd9e1","adjustedStart":"0:00:48.715","adjustedEnd":"0:00:48.757","start":"0:00:48.715","end":"0:00:48.757"}]}],"instances":[{"adjustedStart":"0:00:47.464","adjustedEnd":"0:00:49.299","start":"0:00:47.464","end":"0:00:49.299"}]},{"id":12,"tags":["Wide","LeftFace","Indoor"],"keyFrames":[{"id":19,"instances":[{"thumbnailId":"1c843ee1-f5de-4c22-a401-8521af9313c3","adjustedStart":"0:00:49.299","adjustedEnd":"0:00:49.341","start":"0:00:49.299","end":"0:00:49.341"}]}],"instances":[{"adjustedStart":"0:00:49.299","adjustedEnd":"0:00:57.933","start":"0:00:49.299","end":"0:00:57.933"}]},{"id":13,"tags":["Indoor"],"keyFrames":[{"id":20,"instances":[{"thumbnailId":"a3f461d2-8403-44e4-869d-c619c120b431","adjustedStart":"0:00:57.933","adjustedEnd":"0:00:57.975","start":"0:00:57.933","end":"0:00:57.975"}]},{"id":21,"instances":[{"thumbnailId":"59dc968b-38c2-401e-968c-66d3176e6354","adjustedStart":"0:01:04.606","adjustedEnd":"0:01:04.648","start":"0:01:04.606","end":"0:01:04.648"}]},{"id":22,"instances":[{"thumbnailId":"9bf298af-6e7f-40ab-b08d-ce14cd049216","adjustedStart":"0:01:09.861","adjustedEnd":"0:01:09.903","start":"0:01:09.861","end":"0:01:09.903"}]}],"instances":[{"adjustedStart":"0:00:57.933","adjustedEnd":"0:01:14.783","start":"0:00:57.933","end":"0:01:14.783"}]},{"id":14,"tags":["Indoor"],"keyFrames":[{"id":23,"instances":[{"thumbnailId":"7f995702-d512-43f2-a544-da51b3b9a56e","adjustedStart":"0:01:14.783","adjustedEnd":"0:01:14.825","start":"0:01:14.783","end":"0:01:14.825"}]},{"id":24,"instances":[{"thumbnailId":"796185cc-82c9-4735-8b45-e566d45cb4d9","adjustedStart":"0:01:16.076","adjustedEnd":"0:01:16.118","start":"0:01:16.076","end":"0:01:16.118"}]}],"instances":[{"adjustedStart":"0:01:14.783","adjustedEnd":"0:01:19.454","start":"0:01:14.783","end":"0:01:19.454"}]},{"id":15,"keyFrames":[{"id":25,"instances":[{"thumbnailId":"5f1d49ef-3a89-489e-8bc9-3313ad2a1e6e","adjustedStart":"0:01:19.454","adjustedEnd":"0:01:19.496","start":"0:01:19.454","end":"0:01:19.496"}]}],"instances":[{"adjustedStart":"0:01:19.454","adjustedEnd":"0:01:24.626","start":"0:01:19.454","end":"0:01:24.626"}]},{"id":16,"tags":["CloseUp","CenterFace","Indoor"],"keyFrames":[{"id":26,"instances":[{"thumbnailId":"522d408c-4ebf-4eb1-a563-34590d3a0a98","adjustedStart":"0:01:24.626","adjustedEnd":"0:01:24.668","start":"0:01:24.626","end":"0:01:24.668"}]}],"instances":[{"adjustedStart":"0:01:24.626","adjustedEnd":"0:01:27.296","start":"0:01:24.626","end":"0:01:27.296"}]},{"id":17,"tags":["Medium","LeftFace","Indoor"],"keyFrames":[{"id":27,"instances":[{"thumbnailId":"cf4c5f04-060c-4fbd-85d2-46b68c5812a5","adjustedStart":"0:01:27.296","adjustedEnd":"0:01:27.338","start":"0:01:27.296","end":"0:01:27.338"}]}],"instances":[{"adjustedStart":"0:01:27.296","adjustedEnd":"0:01:42.644","start":"0:01:27.296","end":"0:01:42.644"}]},{"id":18,"tags":["Indoor"],"keyFrames":[{"id":28,"instances":[{"thumbnailId":"314f4828-5e58-44d8-9553-67adfd5775a1","adjustedStart":"0:01:42.644","adjustedEnd":"0:01:42.686","start":"0:01:42.644","end":"0:01:42.686"}]},{"id":29,"instances":[{"thumbnailId":"66790823-fe61-473b-b26a-873c28cc967f","adjustedStart":"0:01:47.441","adjustedEnd":"0:01:47.483","start":"0:01:47.441","end":"0:01:47.483"}]},{"id":30,"instances":[{"thumbnailId":"c4a923f4-da0d-4e65-96e4-92f1ce64607b","adjustedStart":"0:01:50.319","adjustedEnd":"0:01:50.361","start":"0:01:50.319","end":"0:01:50.361"}]}],"instances":[{"adjustedStart":"0:01:42.644","adjustedEnd":"0:01:54.615","start":"0:01:42.644","end":"0:01:54.615"}]},{"id":19,"tags":["Medium","LeftFace","Indoor"],"keyFrames":[{"id":31,"instances":[{"thumbnailId":"55c9be1c-e63b-49d2-ac7e-a97d3f83bba1","adjustedStart":"0:01:54.615","adjustedEnd":"0:01:54.657","start":"0:01:54.615","end":"0:01:54.657"}]},{"id":32,"instances":[{"thumbnailId":"583746f5-2426-4890-a0b1-1fc0c448a3d5","adjustedStart":"0:02:03.415","adjustedEnd":"0:02:03.457","start":"0:02:03.415","end":"0:02:03.457"}]},{"id":33,"instances":[{"thumbnailId":"931cf63b-2807-47e9-a5a9-c0e44c266337","adjustedStart":"0:02:05.292","adjustedEnd":"0:02:05.334","start":"0:02:05.292","end":"0:02:05.334"}]}],"instances":[{"adjustedStart":"0:01:54.615","adjustedEnd":"0:02:06.376","start":"0:01:54.615","end":"0:02:06.376"}]},{"id":20,"tags":["Indoor","Wide","CenterFace"],"keyFrames":[{"id":34,"instances":[{"thumbnailId":"33a6ef5f-796c-4889-9245-012265a7fa20","adjustedStart":"0:02:06.376","adjustedEnd":"0:02:06.418","start":"0:02:06.376","end":"0:02:06.418"}]},{"id":35,"instances":[{"thumbnailId":"48da5607-d1ba-4720-b2ec-5a12ed54a4d7","adjustedStart":"0:02:14.968","adjustedEnd":"0:02:15.01","start":"0:02:14.968","end":"0:02:15.01"}]},{"id":36,"instances":[{"thumbnailId":"75fa2c84-0841-465f-bc4e-ee987def6928","adjustedStart":"0:02:20.14","adjustedEnd":"0:02:20.182","start":"0:02:20.14","end":"0:02:20.182"}]}],"instances":[{"adjustedStart":"0:02:06.376","adjustedEnd":"0:02:22.726","start":"0:02:06.376","end":"0:02:22.726"}]},{"id":21,"tags":["Indoor"],"keyFrames":[{"id":37,"instances":[{"thumbnailId":"d24ac1a7-3afc-49da-a3b8-3362428d5d0b","adjustedStart":"0:02:22.726","adjustedEnd":"0:02:22.768","start":"0:02:22.726","end":"0:02:22.768"}]},{"id":38,"instances":[{"thumbnailId":"5f7db015-9b9f-4947-94b8-6cac0fcdfc2c","adjustedStart":"0:02:23.477","adjustedEnd":"0:02:23.519","start":"0:02:23.477","end":"0:02:23.519"}]}],"instances":[{"adjustedStart":"0:02:22.726","adjustedEnd":"0:02:24.227","start":"0:02:22.726","end":"0:02:24.227"}]},{"id":22,"keyFrames":[{"id":39,"instances":[{"thumbnailId":"b6867baa-7c70-4454-9c6e-0a636811af2e","adjustedStart":"0:02:24.227","adjustedEnd":"0:02:24.269","start":"0:02:24.227","end":"0:02:24.269"}]},{"id":40,"instances":[{"thumbnailId":"22a3648b-1e81-439c-8052-da1e21850ee3","adjustedStart":"0:02:25.02","adjustedEnd":"0:02:25.062","start":"0:02:25.02","end":"0:02:25.062"}]}],"instances":[{"adjustedStart":"0:02:24.227","adjustedEnd":"0:02:26.271","start":"0:02:24.227","end":"0:02:26.271"}]},{"id":23,"keyFrames":[{"id":41,"instances":[{"thumbnailId":"cccdfd00-912f-4c74-af69-86de504323c2","adjustedStart":"0:02:26.271","adjustedEnd":"0:02:26.313","start":"0:02:26.271","end":"0:02:26.313"}]},{"id":42,"instances":[{"thumbnailId":"db37fb3b-2fa2-4374-9bcb-ac6b1cae3e51","adjustedStart":"0:02:27.689","adjustedEnd":"0:02:27.731","start":"0:02:27.689","end":"0:02:27.731"}]}],"instances":[{"adjustedStart":"0:02:26.271","adjustedEnd":"0:02:28.69","start":"0:02:26.271","end":"0:02:28.69"}]},{"id":24,"keyFrames":[{"id":43,"instances":[{"thumbnailId":"5fb90b17-af46-4fab-90e9-ab8a56235d68","adjustedStart":"0:02:28.69","adjustedEnd":"0:02:28.732","start":"0:02:28.69","end":"0:02:28.732"}]},{"id":44,"instances":[{"thumbnailId":"ba9df3de-3c1a-41cd-9629-786606b4dfaa","adjustedStart":"0:02:29.524","adjustedEnd":"0:02:29.566","start":"0:02:29.524","end":"0:02:29.566"}]}],"instances":[{"adjustedStart":"0:02:28.69","adjustedEnd":"0:02:30.776","start":"0:02:28.69","end":"0:02:30.776"}]},{"id":25,"keyFrames":[{"id":45,"instances":[{"thumbnailId":"9cdb0507-7e07-4089-84ac-345f1af971bd","adjustedStart":"0:02:30.776","adjustedEnd":"0:02:30.818","start":"0:02:30.776","end":"0:02:30.818"}]},{"id":46,"instances":[{"thumbnailId":"d7fdee82-7f8f-4c8a-bfad-b4b9efca4f6b","adjustedStart":"0:02:31.735","adjustedEnd":"0:02:31.777","start":"0:02:31.735","end":"0:02:31.777"}]}],"instances":[{"adjustedStart":"0:02:30.776","adjustedEnd":"0:02:32.444","start":"0:02:30.776","end":"0:02:32.444"}]}],"sentiments":[{"id":1,"averageScore":0.5,"sentimentType":"Neutral","instances":[{"adjustedStart":"0:00:00","adjustedEnd":"0:01:26.13","start":"0:00:00","end":"0:01:26.13"},{"adjustedStart":"0:01:34.9","adjustedEnd":"0:02:32.462","start":"0:01:34.9","end":"0:02:32.462"}]},{"id":2,"averageScore":0.1613,"sentimentType":"Negative","instances":[{"adjustedStart":"0:01:26.13","adjustedEnd":"0:01:34.9","start":"0:01:26.13","end":"0:01:34.9"}]}],"emotions":[{"id":1,"type":"Sad","instances":[{"confidence":0.8066,"adjustedStart":"0:01:26.13","adjustedEnd":"0:01:34.9","start":"0:01:26.13","end":"0:01:34.9"}]}],"visualContentModeration":[{"id":1,"adultScore":0,"racyScore":0.8373,"instances":[{"adjustedStart":"0:01:50.319","adjustedEnd":"0:01:54.615","start":"0:01:50.319","end":"0:01:54.615"}]}],"blocks":[{"id":0,"instances":[{"adjustedStart":"0:00:00","adjustedEnd":"0:00:30.18","start":"0:00:00","end":"0:00:30.18"}]},{"id":1,"instances":[{"adjustedStart":"0:00:30.18","adjustedEnd":"0:01:01.76","start":"0:00:30.18","end":"0:01:01.76"}]},{"id":2,"instances":[{"adjustedStart":"0:01:01.76","adjustedEnd":"0:01:34.9","start":"0:01:01.76","end":"0:01:34.9"}]},{"id":3,"instances":[{"adjustedStart":"0:01:34.9","adjustedEnd":"0:02:11.52","start":"0:01:34.9","end":"0:02:11.52"}]},{"id":4,"instances":[{"adjustedStart":"0:02:11.52","adjustedEnd":"0:02:32.462","start":"0:02:11.52","end":"0:02:32.462"}]}],"framePatterns":[{"id":1,"patternType":"RollingCredits","confidence":0.98,"instances":[{"adjustedStart":"0:02:21","adjustedEnd":"0:02:32","start":"0:02:21","end":"0:02:32"}]},{"id":2,"patternType":"Black","confidence":1,"instances":[{"adjustedStart":"0:00:21.647","adjustedEnd":"0:00:21.73","start":"0:00:21.647","end":"0:00:21.73"}]}],"speakers":[{"id":2,"name":"Speaker #2","instances":[{"adjustedStart":"0:00:26.33","adjustedEnd":"0:00:30.18","start":"0:00:26.33","end":"0:00:30.18"}]},{"id":3,"name":"Speaker #3","instances":[{"adjustedStart":"0:00:30.18","adjustedEnd":"0:00:46.91","start":"0:00:30.18","end":"0:00:46.91"}]},{"id":4,"name":"Speaker #4","instances":[{"adjustedStart":"0:00:46.91","adjustedEnd":"0:01:01.76","start":"0:00:46.91","end":"0:01:01.76"}]},{"id":1,"name":"Speaker #1","instances":[{"adjustedStart":"0:01:01.76","adjustedEnd":"0:01:10.95","start":"0:01:01.76","end":"0:01:10.95"}]},{"id":5,"name":"Speaker #5","instances":[{"adjustedStart":"0:01:10.95","adjustedEnd":"0:01:16.61","start":"0:01:10.95","end":"0:01:16.61"}]},{"id":6,"name":"Speaker #6","instances":[{"adjustedStart":"0:01:16.61","adjustedEnd":"0:01:53.11","start":"0:01:16.61","end":"0:01:53.11"}]},{"id":7,"name":"Speaker #7","instances":[{"adjustedStart":"0:01:53.11","adjustedEnd":"0:01:57.38","start":"0:01:53.11","end":"0:01:57.38"}]},{"id":8,"name":"Speaker #8","instances":[{"adjustedStart":"0:02:05.46","adjustedEnd":"0:02:11.52","start":"0:02:05.46","end":"0:02:11.52"}]},{"id":9,"name":"Speaker #9","instances":[{"adjustedStart":"0:02:11.52","adjustedEnd":"0:02:12.86","start":"0:02:11.52","end":"0:02:12.86"}]}],"textualContentModeration":{"id":0,"bannedWordsCount":0,"bannedWordsRatio":0,"instances":[]},"statistics":{"correspondenceCount":8,"speakerTalkToListenRatio":{"1":0.093,"2":0.039,"3":0.169,"4":0.15,"5":0.057,"6":0.37,"7":0.043,"8":0.061,"9":0.013},"speakerLongestMonolog":{"1":9,"2":3,"3":16,"4":14,"5":5,"6":36,"7":4,"8":6,"9":1},"speakerNumberOfFragments":{"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1},"speakerWordCount":{"1":1,"2":5,"3":40,"4":11,"5":11,"6":105,"7":9,"8":9,"9":2}}},"thumbnailId":"6aa5e2d7-6c19-4084-9090-739fb76e1b3a","detectSourceLanguage":false,"languageAutoDetectMode":"None","sourceLanguage":"en-US","sourceLanguages":["en-US"],"language":"en-US","languages":["en-US"],"indexingPreset":"Default","linguisticModelId":"00000000-0000-0000-0000-000000000000","personModelId":"00000000-0000-0000-0000-000000000000","isAdult":false,"publishedUrl":"https://rodmandev.streaming.mediaservices.windows.net/c8d5c3ea-fe53-483d-a291-e50c8bbe6a7e/Oceans%208%202018%20-.ism/manifest(encryption=cbc)","publishedProxyUrl":null,"viewToken":"Bearer=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1cm46bWljcm9zb2Z0OmF6dXJlOm1lZGlhc2VydmljZXM6Y29udGVudGtleWlkZW50aWZpZXIiOiJmZWM1YWMwZS0xYjJjLTRjYjEtYmRhOC03NmE1YWRmNjRmMjIiLCJuYmYiOjE1ODMxMDA2NTgsImV4cCI6MTU4MzE0MzkxOCwiaXNzIjoiaHR0cHM6Ly9icmVha2Rvd24ubWUiLCJhdWQiOiJCcmVha2Rvd25Vc2VyIn0.aeLlAGAKdONsotG1ZjUsKLShA5YhY-I0l7ifQ5rqYGo"}],"videosRanges":[{"videoId":"2a715d49a3","range":{"start":"0:00:00","end":"0:02:32.512"}}]} \ No newline at end of file diff --git a/controllers/oceans8.json b/controllers/oceans8.json new file mode 100644 index 0000000..04007dd --- /dev/null +++ b/controllers/oceans8.json @@ -0,0 +1 @@ +{"partition":null,"description":null,"privacyMode":"Public","state":"Processed","accountId":"5a8c6ddf-900f-48b4-8a3a-0cd5cf48cb4b","id":"d9d45714f3","name":"Ocean 8","userName":"Umang Mathur","created":"2020-04-08T02:57:41.0369176+00:00","isOwned":true,"isEditable":true,"isBase":true,"durationInSeconds":152,"summarizedInsights":{"name":"Ocean 8","id":"d9d45714f3","privacyMode":"Public","duration":{"time":"0:02:32.555","seconds":152.6},"thumbnailVideoId":"d9d45714f3","thumbnailId":"344ad76c-d471-41db-a049-ba8233fe96d3","faces":[{"videoId":"d9d45714f3","referenceId":"02e07452-35e9-a82c-f32e-db9dadf6a616","referenceType":"Bing","confidence":0.8664,"description":"Sandra Annette Bullock is an American-German actress, producer, and philanthropist. She was the highest paid actress in the world in 2010 and 2014. In 2015, Bullock was chosen as People's Most Beautiful Woman and was included in Time's 100 most influential people in the world in 2010. Bullock is the …","title":"American Actress","thumbnailId":"d9cc852e-f5ba-4238-921b-0a00865b6841","seenDuration":84.7,"seenDurationRatio":0.5552,"id":1049,"name":"Sandra Bullock","appearances":[{"startTime":"0:00:05.589","endTime":"0:00:56.181","startSeconds":5.6,"endSeconds":56.2},{"startTime":"0:01:24.585","endTime":"0:01:54.323","startSeconds":84.6,"endSeconds":114.3},{"startTime":"0:02:14.468","endTime":"0:02:18.931","startSeconds":134.5,"endSeconds":138.9}]},{"videoId":"d9d45714f3","confidence":0,"description":null,"title":null,"thumbnailId":"9f079bb6-b60b-4431-baa7-ca0bfa993314","seenDuration":14.2,"seenDurationRatio":0.0931,"id":1308,"name":"Unknown #2","appearances":[{"startTime":"0:00:34.701","endTime":"0:00:48.924","startSeconds":34.7,"endSeconds":48.9}]},{"videoId":"d9d45714f3","confidence":0,"description":null,"title":null,"thumbnailId":"d008f274-463a-422f-a2f5-fa4708d33533","seenDuration":10.9,"seenDurationRatio":0.0714,"id":1621,"name":"Unknown #1","appearances":[{"startTime":"0:01:55.616","endTime":"0:02:05.25","startSeconds":115.6,"endSeconds":125.2},{"startTime":"0:02:21.141","endTime":"0:02:22.434","startSeconds":141.1,"endSeconds":142.4}]},{"videoId":"d9d45714f3","confidence":0,"description":null,"title":null,"thumbnailId":"60831835-6654-4805-a254-f239db6f4dc1","seenDuration":2.2,"seenDurationRatio":0.0144,"id":1083,"name":"Unknown #4","appearances":[{"startTime":"0:00:24.942","endTime":"0:00:27.069","startSeconds":24.9,"endSeconds":27.1}]},{"videoId":"d9d45714f3","confidence":0,"description":null,"title":null,"thumbnailId":"c7c0b07e-d760-43c3-9d5c-8b3413883e58","seenDuration":1.2,"seenDurationRatio":0.0079,"id":1089,"name":"Unknown #6","appearances":[{"startTime":"0:00:27.194","endTime":"0:00:28.404","startSeconds":27.2,"endSeconds":28.4}]}],"keywords":[{"isTranscript":true,"id":2,"name":"monica","appearances":[{"startTime":"0:01:08.73","endTime":"0:01:16.92","startSeconds":68.7,"endSeconds":76.9},{"startTime":"0:01:20.94","endTime":"0:01:27.33","startSeconds":80.9,"endSeconds":87.3}]},{"isTranscript":true,"id":3,"name":"back","appearances":[{"startTime":"0:01:34.69","endTime":"0:01:41.43","startSeconds":94.7,"endSeconds":101.4}]},{"isTranscript":false,"id":1,"name":"clips","appearances":[{"startTime":"0:00:08.258","endTime":"0:00:08.717","startSeconds":8.3,"endSeconds":8.7},{"startTime":"0:00:15.599","endTime":"0:00:17.893","startSeconds":15.6,"endSeconds":17.9},{"startTime":"0:02:01.121","endTime":"0:02:01.58","startSeconds":121.1,"endSeconds":121.6},{"startTime":"0:02:24.061","endTime":"0:02:30.526","startSeconds":144.1,"endSeconds":150.5}]},{"isTranscript":false,"id":4,"name":"movieclips","appearances":[{"startTime":"0:00:08.258","endTime":"0:00:08.717","startSeconds":8.3,"endSeconds":8.7},{"startTime":"0:02:01.121","endTime":"0:02:01.58","startSeconds":121.1,"endSeconds":121.6},{"startTime":"0:02:26.355","endTime":"0:02:28.649","startSeconds":146.4,"endSeconds":148.6}]},{"isTranscript":false,"id":5,"name":"eclips","appearances":[{"startTime":"0:00:08.258","endTime":"0:00:08.717","startSeconds":8.3,"endSeconds":8.7},{"startTime":"0:00:15.599","endTime":"0:00:17.893","startSeconds":15.6,"endSeconds":17.9},{"startTime":"0:02:01.121","endTime":"0:02:01.58","startSeconds":121.1,"endSeconds":121.6},{"startTime":"0:02:26.355","endTime":"0:02:28.649","startSeconds":146.4,"endSeconds":148.6}]}],"sentiments":[{"sentimentKey":"Neutral","seenDurationRatio":0.9734,"appearances":[{"startTime":"0:00:00","endTime":"0:01:41.43","startSeconds":0,"endSeconds":101.4},{"startTime":"0:01:45.44","endTime":"0:02:32.509","startSeconds":105.4,"endSeconds":152.5}]},{"sentimentKey":"Positive","seenDurationRatio":0.0374,"appearances":[{"startTime":"0:01:41.43","endTime":"0:01:45.44","startSeconds":101.4,"endSeconds":105.4},{"startTime":"0:02:09.67","endTime":"0:02:11.41","startSeconds":129.7,"endSeconds":131.4}]}],"emotions":[{"type":"Joy","seenDurationRatio":0.0374,"appearances":[{"startTime":"0:01:41.43","endTime":"0:01:45.44","startSeconds":101.4,"endSeconds":105.4},{"startTime":"0:02:09.67","endTime":"0:02:11.41","startSeconds":129.7,"endSeconds":131.4}]}],"audioEffects":[],"labels":[{"id":6,"name":"person","appearances":[{"startTime":"0:00:05.339","endTime":"0:01:16.076","startSeconds":5.3,"endSeconds":76.1},{"startTime":"0:01:20.08","endTime":"0:01:41.435","startSeconds":80.1,"endSeconds":101.4},{"startTime":"0:01:48.108","endTime":"0:02:18.806","startSeconds":108.1,"endSeconds":138.8}]},{"id":7,"name":"indoor","appearances":[{"startTime":"0:00:05.339","endTime":"0:01:18.746","startSeconds":5.3,"endSeconds":78.7},{"startTime":"0:01:34.761","endTime":"0:01:37.431","startSeconds":94.8,"endSeconds":97.4},{"startTime":"0:01:41.435","endTime":"0:02:22.81","startSeconds":101.4,"endSeconds":142.8}]},{"id":10,"name":"woman","appearances":[{"startTime":"0:00:05.339","endTime":"0:00:10.678","startSeconds":5.3,"endSeconds":10.7},{"startTime":"0:00:14.681","endTime":"0:00:57.391","startSeconds":14.7,"endSeconds":57.4},{"startTime":"0:01:01.395","endTime":"0:01:10.738","startSeconds":61.4,"endSeconds":70.7},{"startTime":"0:01:17.411","endTime":"0:01:18.746","startSeconds":77.4,"endSeconds":78.7},{"startTime":"0:01:25.419","endTime":"0:01:41.435","startSeconds":85.4,"endSeconds":101.4},{"startTime":"0:01:45.439","endTime":"0:01:46.774","startSeconds":105.4,"endSeconds":106.8},{"startTime":"0:01:50.777","endTime":"0:02:20.14","startSeconds":110.8,"endSeconds":140.1}]},{"id":2,"name":"wall","appearances":[{"startTime":"0:00:00","endTime":"0:00:01.335","startSeconds":0,"endSeconds":1.3},{"startTime":"0:00:06.673","endTime":"0:00:10.678","startSeconds":6.7,"endSeconds":10.7},{"startTime":"0:00:14.681","endTime":"0:00:21.355","startSeconds":14.7,"endSeconds":21.4},{"startTime":"0:00:25.359","endTime":"0:00:57.391","startSeconds":25.4,"endSeconds":57.4},{"startTime":"0:01:02.729","endTime":"0:01:16.076","startSeconds":62.7,"endSeconds":76.1},{"startTime":"0:01:41.435","endTime":"0:02:22.81","startSeconds":101.4,"endSeconds":142.8}]},{"id":8,"name":"clothing","appearances":[{"startTime":"0:00:05.339","endTime":"0:00:09.343","startSeconds":5.3,"endSeconds":9.3},{"startTime":"0:00:21.355","endTime":"0:00:57.391","startSeconds":21.4,"endSeconds":57.4},{"startTime":"0:01:01.395","endTime":"0:01:14.742","startSeconds":61.4,"endSeconds":74.7},{"startTime":"0:01:20.08","endTime":"0:01:41.435","startSeconds":80.1,"endSeconds":101.4},{"startTime":"0:01:48.108","endTime":"0:01:53.447","startSeconds":108.1,"endSeconds":113.4},{"startTime":"0:02:00.12","endTime":"0:02:01.455","startSeconds":120.1,"endSeconds":121.5},{"startTime":"0:02:06.793","endTime":"0:02:18.806","startSeconds":126.8,"endSeconds":138.8}]},{"id":1,"name":"screenshot","appearances":[{"startTime":"0:00:00","endTime":"0:00:10.678","startSeconds":0,"endSeconds":10.7},{"startTime":"0:00:16.016","endTime":"0:00:29.363","startSeconds":16,"endSeconds":29.4},{"startTime":"0:00:37.371","endTime":"0:00:48.048","startSeconds":37.4,"endSeconds":48},{"startTime":"0:00:57.391","endTime":"0:01:04.064","startSeconds":57.4,"endSeconds":64.1},{"startTime":"0:01:12.072","endTime":"0:01:13.407","startSeconds":72.1,"endSeconds":73.4},{"startTime":"0:01:21.415","endTime":"0:01:38.766","startSeconds":81.4,"endSeconds":98.8},{"startTime":"0:01:45.439","endTime":"0:01:54.782","startSeconds":105.4,"endSeconds":114.8},{"startTime":"0:02:01.455","endTime":"0:02:12.132","startSeconds":121.5,"endSeconds":132.1},{"startTime":"0:02:22.809","endTime":"0:02:32.444","startSeconds":142.8,"endSeconds":152.4}]},{"id":11,"name":"human face","appearances":[{"startTime":"0:00:05.339","endTime":"0:00:10.678","startSeconds":5.3,"endSeconds":10.7},{"startTime":"0:00:14.681","endTime":"0:00:57.391","startSeconds":14.7,"endSeconds":57.4},{"startTime":"0:01:25.419","endTime":"0:01:41.435","startSeconds":85.4,"endSeconds":101.4},{"startTime":"0:01:50.777","endTime":"0:01:54.782","startSeconds":110.8,"endSeconds":114.8},{"startTime":"0:02:00.12","endTime":"0:02:04.124","startSeconds":120.1,"endSeconds":124.1},{"startTime":"0:02:08.128","endTime":"0:02:12.132","startSeconds":128.1,"endSeconds":132.1},{"startTime":"0:02:17.471","endTime":"0:02:18.806","startSeconds":137.5,"endSeconds":138.8}]},{"id":12,"name":"girl","appearances":[{"startTime":"0:00:06.673","endTime":"0:00:10.678","startSeconds":6.7,"endSeconds":10.7},{"startTime":"0:00:14.681","endTime":"0:00:28.028","startSeconds":14.7,"endSeconds":28},{"startTime":"0:00:33.367","endTime":"0:00:48.048","startSeconds":33.4,"endSeconds":48},{"startTime":"0:01:25.419","endTime":"0:01:41.435","startSeconds":85.4,"endSeconds":101.4},{"startTime":"0:01:45.439","endTime":"0:01:54.782","startSeconds":105.4,"endSeconds":114.8},{"startTime":"0:02:00.12","endTime":"0:02:04.124","startSeconds":120.1,"endSeconds":124.1},{"startTime":"0:02:08.128","endTime":"0:02:13.467","startSeconds":128.1,"endSeconds":133.5},{"startTime":"0:02:17.471","endTime":"0:02:18.806","startSeconds":137.5,"endSeconds":138.8}]},{"id":9,"name":"hair","appearances":[{"startTime":"0:00:05.339","endTime":"0:00:10.678","startSeconds":5.3,"endSeconds":10.7},{"startTime":"0:00:14.681","endTime":"0:00:29.363","startSeconds":14.7,"endSeconds":29.4},{"startTime":"0:00:34.701","endTime":"0:00:49.383","startSeconds":34.7,"endSeconds":49.4},{"startTime":"0:01:25.419","endTime":"0:01:28.088","startSeconds":85.4,"endSeconds":88.1},{"startTime":"0:01:46.773","endTime":"0:01:54.782","startSeconds":106.8,"endSeconds":114.8},{"startTime":"0:01:58.785","endTime":"0:02:05.459","startSeconds":118.8,"endSeconds":125.5},{"startTime":"0:02:18.805","endTime":"0:02:20.14","startSeconds":138.8,"endSeconds":140.1}]},{"id":17,"name":"mirror","appearances":[{"startTime":"0:00:20.02","endTime":"0:00:21.355","startSeconds":20,"endSeconds":21.4},{"startTime":"0:00:37.371","endTime":"0:00:38.706","startSeconds":37.4,"endSeconds":38.7},{"startTime":"0:00:45.379","endTime":"0:00:46.714","startSeconds":45.4,"endSeconds":46.7},{"startTime":"0:01:02.729","endTime":"0:01:08.068","startSeconds":62.7,"endSeconds":68.1},{"startTime":"0:01:44.104","endTime":"0:01:54.782","startSeconds":104.1,"endSeconds":114.8}]},{"id":19,"name":"standing","appearances":[{"startTime":"0:00:26.693","endTime":"0:00:28.028","startSeconds":26.7,"endSeconds":28},{"startTime":"0:00:33.367","endTime":"0:00:34.702","startSeconds":33.4,"endSeconds":34.7},{"startTime":"0:00:49.383","endTime":"0:00:50.718","startSeconds":49.4,"endSeconds":50.7},{"startTime":"0:01:20.08","endTime":"0:01:25.419","startSeconds":80.1,"endSeconds":85.4},{"startTime":"0:02:06.793","endTime":"0:02:09.463","startSeconds":126.8,"endSeconds":129.5},{"startTime":"0:02:13.467","endTime":"0:02:18.806","startSeconds":133.5,"endSeconds":138.8}]},{"id":18,"name":"text","appearances":[{"startTime":"0:00:25.359","endTime":"0:00:28.028","startSeconds":25.4,"endSeconds":28},{"startTime":"0:00:36.036","endTime":"0:00:37.371","startSeconds":36,"endSeconds":37.4},{"startTime":"0:00:44.044","endTime":"0:00:45.379","startSeconds":44,"endSeconds":45.4},{"startTime":"0:01:18.745","endTime":"0:01:20.08","startSeconds":78.7,"endSeconds":80.1},{"startTime":"0:02:22.809","endTime":"0:02:32.444","startSeconds":142.8,"endSeconds":152.4}]},{"id":55,"name":"dress","appearances":[{"startTime":"0:01:29.423","endTime":"0:01:30.758","startSeconds":89.4,"endSeconds":90.8},{"startTime":"0:02:06.793","endTime":"0:02:18.806","startSeconds":126.8,"endSeconds":138.8}]},{"id":53,"name":"sitting","appearances":[{"startTime":"0:01:28.088","endTime":"0:01:41.435","startSeconds":88.1,"endSeconds":101.4}]},{"id":28,"name":"design","appearances":[{"startTime":"0:00:54.721","endTime":"0:00:57.391","startSeconds":54.7,"endSeconds":57.4},{"startTime":"0:02:22.809","endTime":"0:02:32.444","startSeconds":142.8,"endSeconds":152.4}]},{"id":35,"name":"room","appearances":[{"startTime":"0:01:05.399","endTime":"0:01:06.734","startSeconds":65.4,"endSeconds":66.7},{"startTime":"0:01:14.741","endTime":"0:01:16.076","startSeconds":74.7,"endSeconds":76.1},{"startTime":"0:02:05.459","endTime":"0:02:12.132","startSeconds":125.5,"endSeconds":132.1},{"startTime":"0:02:20.14","endTime":"0:02:21.475","startSeconds":140.1,"endSeconds":141.5}]},{"id":40,"name":"display","appearances":[{"startTime":"0:01:16.076","endTime":"0:01:17.411","startSeconds":76.1,"endSeconds":77.4},{"startTime":"0:02:24.144","endTime":"0:02:32.152","startSeconds":144.1,"endSeconds":152.2}]},{"id":70,"name":"screen","appearances":[{"startTime":"0:02:24.144","endTime":"0:02:32.152","startSeconds":144.1,"endSeconds":152.2}]},{"id":71,"name":"monitor","appearances":[{"startTime":"0:02:24.144","endTime":"0:02:32.152","startSeconds":144.1,"endSeconds":152.2}]},{"id":38,"name":"man","appearances":[{"startTime":"0:01:12.072","endTime":"0:01:14.742","startSeconds":72.1,"endSeconds":74.7},{"startTime":"0:01:20.08","endTime":"0:01:25.419","startSeconds":80.1,"endSeconds":85.4}]},{"id":34,"name":"door","appearances":[{"startTime":"0:01:04.064","endTime":"0:01:05.399","startSeconds":64.1,"endSeconds":65.4},{"startTime":"0:01:49.443","endTime":"0:01:54.782","startSeconds":109.4,"endSeconds":114.8}]},{"id":25,"name":"computer","appearances":[{"startTime":"0:00:34.701","endTime":"0:00:41.375","startSeconds":34.7,"endSeconds":41.4}]},{"id":16,"name":"face","appearances":[{"startTime":"0:00:16.016","endTime":"0:00:20.02","startSeconds":16,"endSeconds":20},{"startTime":"0:01:25.419","endTime":"0:01:28.088","startSeconds":85.4,"endSeconds":88.1}]},{"id":33,"name":"video game","appearances":[{"startTime":"0:01:01.395","endTime":"0:01:02.73","startSeconds":61.4,"endSeconds":62.7},{"startTime":"0:01:12.072","endTime":"0:01:13.407","startSeconds":72.1,"endSeconds":73.4},{"startTime":"0:01:20.08","endTime":"0:01:22.75","startSeconds":80.1,"endSeconds":82.8}]},{"id":41,"name":"car","appearances":[{"startTime":"0:01:20.08","endTime":"0:01:25.419","startSeconds":80.1,"endSeconds":85.4}]},{"id":42,"name":"vehicle","appearances":[{"startTime":"0:01:20.08","endTime":"0:01:25.419","startSeconds":80.1,"endSeconds":85.4}]},{"id":5,"name":"shelf","appearances":[{"startTime":"0:00:04.004","endTime":"0:00:05.339","startSeconds":4,"endSeconds":5.3},{"startTime":"0:00:12.012","endTime":"0:00:14.682","startSeconds":12,"endSeconds":14.7},{"startTime":"0:01:16.076","endTime":"0:01:17.411","startSeconds":76.1,"endSeconds":77.4}]},{"id":21,"name":"furniture","appearances":[{"startTime":"0:00:29.363","endTime":"0:00:30.698","startSeconds":29.4,"endSeconds":30.7},{"startTime":"0:00:52.052","endTime":"0:00:54.722","startSeconds":52.1,"endSeconds":54.7},{"startTime":"0:01:56.116","endTime":"0:01:57.451","startSeconds":116.1,"endSeconds":117.5}]},{"id":26,"name":"kitchen","appearances":[{"startTime":"0:00:34.701","endTime":"0:00:40.04","startSeconds":34.7,"endSeconds":40}]},{"id":39,"name":"suit","appearances":[{"startTime":"0:01:13.407","endTime":"0:01:14.742","startSeconds":73.4,"endSeconds":74.7},{"startTime":"0:01:21.415","endTime":"0:01:25.419","startSeconds":81.4,"endSeconds":85.4}]},{"id":73,"name":"template","appearances":[{"startTime":"0:02:25.479","endTime":"0:02:29.483","startSeconds":145.5,"endSeconds":149.5}]},{"id":52,"name":"portrait","appearances":[{"startTime":"0:01:25.419","endTime":"0:01:28.088","startSeconds":85.4,"endSeconds":88.1},{"startTime":"0:02:02.789","endTime":"0:02:04.124","startSeconds":122.8,"endSeconds":124.1}]},{"id":69,"name":"bed","appearances":[{"startTime":"0:02:20.14","endTime":"0:02:22.81","startSeconds":140.1,"endSeconds":142.8}]},{"id":24,"name":"ceiling","appearances":[{"startTime":"0:00:33.367","endTime":"0:00:34.702","startSeconds":33.4,"endSeconds":34.7},{"startTime":"0:00:54.721","endTime":"0:00:56.056","startSeconds":54.7,"endSeconds":56.1}]},{"id":30,"name":"art","appearances":[{"startTime":"0:00:57.391","endTime":"0:01:00.06","startSeconds":57.4,"endSeconds":60.1}]},{"id":45,"name":"business","appearances":[{"startTime":"0:01:22.749","endTime":"0:01:25.419","startSeconds":82.7,"endSeconds":85.4}]},{"id":23,"name":"people","appearances":[{"startTime":"0:00:32.032","endTime":"0:00:33.367","startSeconds":32,"endSeconds":33.4},{"startTime":"0:01:05.399","endTime":"0:01:06.734","startSeconds":65.4,"endSeconds":66.7}]},{"id":50,"name":"lady","appearances":[{"startTime":"0:01:25.419","endTime":"0:01:28.088","startSeconds":85.4,"endSeconds":88.1}]},{"id":65,"name":"window","appearances":[{"startTime":"0:02:18.805","endTime":"0:02:21.475","startSeconds":138.8,"endSeconds":141.5}]},{"id":27,"name":"appliance","appearances":[{"startTime":"0:00:53.387","endTime":"0:00:54.722","startSeconds":53.4,"endSeconds":54.7},{"startTime":"0:01:57.451","endTime":"0:01:58.786","startSeconds":117.5,"endSeconds":118.8}]},{"id":3,"name":"keyboard","appearances":[{"startTime":"0:00:02.669","endTime":"0:00:05.339","startSeconds":2.7,"endSeconds":5.3}]},{"id":13,"name":"long hair","appearances":[{"startTime":"0:00:08.008","endTime":"0:00:09.343","startSeconds":8,"endSeconds":9.3},{"startTime":"0:00:38.705","endTime":"0:00:40.04","startSeconds":38.7,"endSeconds":40}]},{"id":20,"name":"table","appearances":[{"startTime":"0:00:29.363","endTime":"0:00:30.698","startSeconds":29.4,"endSeconds":30.7},{"startTime":"0:00:52.052","endTime":"0:00:53.387","startSeconds":52.1,"endSeconds":53.4}]},{"id":60,"name":"dark","appearances":[{"startTime":"0:02:04.124","endTime":"0:02:05.459","startSeconds":124.1,"endSeconds":125.5}]},{"id":61,"name":"playing","appearances":[{"startTime":"0:02:08.128","endTime":"0:02:09.463","startSeconds":128.1,"endSeconds":129.5}]},{"id":63,"name":"smoke","appearances":[{"startTime":"0:02:16.136","endTime":"0:02:17.471","startSeconds":136.1,"endSeconds":137.5}]},{"id":64,"name":"nature","appearances":[{"startTime":"0:02:16.136","endTime":"0:02:17.471","startSeconds":136.1,"endSeconds":137.5}]},{"id":66,"name":"house","appearances":[{"startTime":"0:02:20.14","endTime":"0:02:21.475","startSeconds":140.1,"endSeconds":141.5}]},{"id":67,"name":"pillow","appearances":[{"startTime":"0:02:20.14","endTime":"0:02:21.475","startSeconds":140.1,"endSeconds":141.5}]},{"id":68,"name":"bedroom","appearances":[{"startTime":"0:02:20.14","endTime":"0:02:21.475","startSeconds":140.1,"endSeconds":141.5}]},{"id":72,"name":"remote","appearances":[{"startTime":"0:02:24.144","endTime":"0:02:25.479","startSeconds":144.1,"endSeconds":145.5}]},{"id":29,"name":"glasses","appearances":[{"startTime":"0:00:54.721","endTime":"0:00:56.056","startSeconds":54.7,"endSeconds":56.1}]},{"id":31,"name":"bag","appearances":[{"startTime":"0:00:58.725","endTime":"0:01:00.06","startSeconds":58.7,"endSeconds":60.1}]},{"id":32,"name":"accessory","appearances":[{"startTime":"0:00:58.725","endTime":"0:01:00.06","startSeconds":58.7,"endSeconds":60.1}]},{"id":43,"name":"land vehicle","appearances":[{"startTime":"0:01:21.415","endTime":"0:01:22.75","startSeconds":81.4,"endSeconds":82.8}]},{"id":44,"name":"walking","appearances":[{"startTime":"0:01:21.415","endTime":"0:01:22.75","startSeconds":81.4,"endSeconds":82.8}]},{"id":47,"name":"phone","appearances":[{"startTime":"0:01:25.419","endTime":"0:01:26.754","startSeconds":85.4,"endSeconds":86.8}]},{"id":48,"name":"using","appearances":[{"startTime":"0:01:25.419","endTime":"0:01:26.754","startSeconds":85.4,"endSeconds":86.8}]},{"id":49,"name":"talking","appearances":[{"startTime":"0:01:25.419","endTime":"0:01:26.754","startSeconds":85.4,"endSeconds":86.8}]},{"id":51,"name":"cellphone","appearances":[{"startTime":"0:01:25.419","endTime":"0:01:26.754","startSeconds":85.4,"endSeconds":86.8}]},{"id":54,"name":"fashion","appearances":[{"startTime":"0:01:29.423","endTime":"0:01:30.758","startSeconds":89.4,"endSeconds":90.8}]},{"id":59,"name":"looking","appearances":[{"startTime":"0:01:49.443","endTime":"0:01:50.778","startSeconds":109.4,"endSeconds":110.8}]},{"id":75,"name":"internet","appearances":[{"startTime":"0:02:30.817","endTime":"0:02:32.152","startSeconds":150.8,"endSeconds":152.2}]},{"id":46,"name":"wearing","appearances":[{"startTime":"0:01:24.084","endTime":"0:01:25.419","startSeconds":84.1,"endSeconds":85.4}]},{"id":14,"name":"food","appearances":[{"startTime":"0:00:12.012","endTime":"0:00:13.347","startSeconds":12,"endSeconds":13.3}]},{"id":15,"name":"bottle","appearances":[{"startTime":"0:00:12.012","endTime":"0:00:13.347","startSeconds":12,"endSeconds":13.3}]},{"id":22,"name":"whiteboard","appearances":[{"startTime":"0:00:29.363","endTime":"0:00:30.698","startSeconds":29.4,"endSeconds":30.7}]},{"id":4,"name":"close","appearances":[{"startTime":"0:00:02.669","endTime":"0:00:04.004","startSeconds":2.7,"endSeconds":4}]},{"id":36,"name":"flower","appearances":[{"startTime":"0:01:09.403","endTime":"0:01:10.738","startSeconds":69.4,"endSeconds":70.7}]},{"id":37,"name":"vase","appearances":[{"startTime":"0:01:09.403","endTime":"0:01:10.738","startSeconds":69.4,"endSeconds":70.7}]},{"id":56,"name":"footwear","appearances":[{"startTime":"0:01:34.761","endTime":"0:01:36.096","startSeconds":94.8,"endSeconds":96.1}]},{"id":57,"name":"clothes hanger","appearances":[{"startTime":"0:01:42.769","endTime":"0:01:44.104","startSeconds":102.8,"endSeconds":104.1}]},{"id":58,"name":"hanger","appearances":[{"startTime":"0:01:42.769","endTime":"0:01:44.104","startSeconds":102.8,"endSeconds":104.1}]},{"id":62,"name":"white","appearances":[{"startTime":"0:02:14.801","endTime":"0:02:16.136","startSeconds":134.8,"endSeconds":136.1}]},{"id":74,"name":"electronics","appearances":[{"startTime":"0:02:26.813","endTime":"0:02:28.148","startSeconds":146.8,"endSeconds":148.1}]}],"framePatterns":[{"id":1,"name":"RollingCredits","appearances":[{"startTime":"0:02:21","endTime":"0:02:32","startSeconds":141,"endSeconds":152}]},{"id":2,"name":"Black","appearances":[{"startTime":"0:00:21.647","endTime":"0:00:21.73","startSeconds":21.6,"endSeconds":21.7}]}],"brands":[],"namedLocations":[],"namedPeople":[],"statistics":{"correspondenceCount":8,"speakerTalkToListenRatio":{"1":0.255,"2":0.02,"3":0.376,"4":0.102,"5":0.14,"6":0.004,"7":0.083,"8":0.017},"speakerLongestMonolog":{"1":16,"2":1,"3":13,"4":6,"5":9,"6":0,"7":5,"8":1},"speakerNumberOfFragments":{"1":1,"2":1,"3":3,"4":1,"5":1,"6":1,"7":1,"8":1},"speakerWordCount":{"1":57,"2":6,"3":69,"4":19,"5":33,"6":1,"7":19,"8":4}},"topics":[]},"videos":[{"accountId":"5a8c6ddf-900f-48b4-8a3a-0cd5cf48cb4b","id":"d9d45714f3","state":"Processed","moderationState":"OK","reviewState":"None","privacyMode":"Public","processingProgress":"100%","failureCode":"None","failureMessage":"","externalId":null,"externalUrl":null,"metadata":null,"insights":{"version":"1.0.0.0","duration":"0:02:32.555","sourceLanguage":"en-US","sourceLanguages":["en-US"],"language":"en-US","languages":["en-US"],"transcript":[{"id":1,"text":"Hi, I'd like to return of course received no but they are unopened.","confidence":0.6989,"speakerId":1,"language":"en-US","instances":[{"adjustedStart":"0:00:29.73","adjustedEnd":"0:00:33.79","start":"0:00:29.73","end":"0:00:33.79"}]},{"id":2,"text":"They haven't touched. I really need your receipt.","confidence":0.6989,"speakerId":1,"language":"en-US","instances":[{"adjustedStart":"0:00:33.79","adjustedEnd":"0:00:36.88","start":"0:00:33.79","end":"0:00:36.88"}]},{"id":3,"text":"They are sealed, their brand new.","confidence":0.8781,"speakerId":1,"language":"en-US","instances":[{"adjustedStart":"0:00:36.88","adjustedEnd":"0:00:38.51","start":"0:00:36.88","end":"0:00:38.51"}]},{"id":4,"text":"Do you have the credit card that you use?","confidence":0.8781,"speakerId":1,"language":"en-US","instances":[{"adjustedStart":"0:00:38.51","adjustedEnd":"0:00:40.95","start":"0:00:38.51","end":"0:00:40.95"}]},{"id":5,"text":"This is ridiculous. I bought these last week.","confidence":0.8781,"speakerId":1,"language":"en-US","instances":[{"adjustedStart":"0:00:40.95","adjustedEnd":"0:00:43.11","start":"0:00:40.95","end":"0:00:43.11"}]},{"id":6,"text":"Maybe you can try client services on the six floor.","confidence":0.8781,"speakerId":1,"language":"en-US","instances":[{"adjustedStart":"0:00:43.11","adjustedEnd":"0:00:45.82","start":"0:00:43.11","end":"0:00:45.82"}]},{"id":7,"text":"No, never mind.","confidence":0.8781,"speakerId":1,"language":"en-US","instances":[{"adjustedStart":"0:00:45.82","adjustedEnd":"0:00:46.64","start":"0:00:45.82","end":"0:00:46.64"}]},{"id":8,"text":"And at least get a bag.","confidence":0.6658,"speakerId":2,"language":"en-US","instances":[{"adjustedStart":"0:00:49.57","adjustedEnd":"0:00:50.9","start":"0:00:49.57","end":"0:00:50.9"}]},{"id":9,"text":"Very random. Traffic. My name is Monica or anything else I can do for you or arrange transportation.","confidence":0.8173,"speakerId":3,"language":"en-US","instances":[{"adjustedStart":"0:01:08.73","adjustedEnd":"0:01:16.92","start":"0:01:08.73","end":"0:01:16.92"}]},{"id":10,"text":"Hi, this is Mrs Randall.","confidence":0.8478,"speakerId":3,"language":"en-US","instances":[{"adjustedStart":"0:01:19.16","adjustedEnd":"0:01:20.94","start":"0:01:19.16","end":"0:01:20.94"}]},{"id":11,"text":"We just checked out of room 2814 nasb Quick Monica please","confidence":0.8478,"speakerId":3,"language":"en-US","instances":[{"adjustedStart":"0:01:20.94","adjustedEnd":"0:01:24.84","start":"0:01:20.94","end":"0:01:24.84"}]},{"id":12,"text":"this is she hi Monica Hi Hi,","confidence":0.8478,"speakerId":4,"language":"en-US","instances":[{"adjustedStart":"0:01:24.84","adjustedEnd":"0:01:27.33","start":"0:01:24.84","end":"0:01:27.33"}]},{"id":13,"text":"we just found out our flight was cancelled.","confidence":0.8478,"speakerId":4,"language":"en-US","instances":[{"adjustedStart":"0:01:27.33","adjustedEnd":"0:01:30.17","start":"0:01:27.33","end":"0:01:30.17"}]},{"id":14,"text":"I know I know.","confidence":0.8478,"speakerId":4,"language":"en-US","instances":[{"adjustedStart":"0:01:30.17","adjustedEnd":"0:01:31.59","start":"0:01:30.17","end":"0:01:31.59"}]},{"id":15,"text":"Instead of seeing some horrible airport hotel,","confidence":0.9081,"speakerId":5,"language":"en-US","instances":[{"adjustedStart":"0:01:32.72","adjustedEnd":"0:01:34.69","start":"0:01:32.72","end":"0:01:34.69"}]},{"id":16,"text":"we were hoping that maybe we could get our original room back.","confidence":0.9081,"speakerId":5,"language":"en-US","instances":[{"adjustedStart":"0:01:34.69","adjustedEnd":"0:01:38.06","start":"0:01:34.69","end":"0:01:38.06"}]},{"id":17,"text":"We can give you that.","confidence":0.9081,"speakerId":5,"language":"en-US","instances":[{"adjustedStart":"0:01:38.06","adjustedEnd":"0:01:39.46","start":"0:01:38.06","end":"0:01:39.46"}]},{"id":18,"text":"Same them back. That would be amazing.","confidence":0.9081,"speakerId":5,"language":"en-US","instances":[{"adjustedStart":"0:01:39.46","adjustedEnd":"0:01:41.43","start":"0:01:39.46","end":"0:01:41.43"}]},{"id":19,"text":"Thank you","confidence":0.9081,"speakerId":5,"language":"en-US","instances":[{"adjustedStart":"0:01:41.43","adjustedEnd":"0:01:41.99","start":"0:01:41.43","end":"0:01:41.99"}]},{"id":20,"text":"so much. Going to grab a bite.","confidence":0.8924,"speakerId":3,"language":"en-US","instances":[{"adjustedStart":"0:01:41.99","adjustedEnd":"0:01:45.44","start":"0:01:41.99","end":"0:01:45.44"}]},{"id":21,"text":"Would it be possible to get the made in there now?","confidence":0.8861,"speakerId":3,"language":"en-US","instances":[{"adjustedStart":"0:01:45.44","adjustedEnd":"0:01:48.42","start":"0:01:45.44","end":"0:01:48.42"}]},{"id":22,"text":"Yes. Absolutely perfectly. Really appreciate it.","confidence":0.8861,"speakerId":3,"language":"en-US","instances":[{"adjustedStart":"0:01:48.42","adjustedEnd":"0:01:50.04","start":"0:01:48.42","end":"0:01:50.04"}]},{"id":23,"text":"It's not a problem. Thank you so much.","confidence":0.8861,"speakerId":3,"language":"en-US","instances":[{"adjustedStart":"0:01:50.04","adjustedEnd":"0:01:52.21","start":"0:01:50.04","end":"0:01:52.21"}]},{"id":24,"text":"Will see you","confidence":0.8861,"speakerId":3,"language":"en-US","instances":[{"adjustedStart":"0:01:52.21","adjustedEnd":"0:01:53.02","start":"0:01:52.21","end":"0:01:53.02"}]},{"id":25,"text":"soon.","confidence":0.8861,"speakerId":6,"language":"en-US","instances":[{"adjustedStart":"0:01:53.02","adjustedEnd":"0:01:53.29","start":"0:01:53.02","end":"0:01:53.29"}]},{"id":26,"text":"you finish up later? I just have to get off my feet course.","confidence":0.7605,"speakerId":7,"language":"en-US","instances":[{"adjustedStart":"0:02:05.88","adjustedEnd":"0:02:09.67","start":"0:02:05.88","end":"0:02:09.67"}]},{"id":27,"text":"Thank you so much. Have a","confidence":0.7605,"speakerId":7,"language":"en-US","instances":[{"adjustedStart":"0:02:09.67","adjustedEnd":"0:02:11.41","start":"0:02:09.67","end":"0:02:11.41"}]},{"id":28,"text":"great day. Bye bye.","confidence":0.7605,"speakerId":8,"language":"en-US","instances":[{"adjustedStart":"0:02:11.41","adjustedEnd":"0:02:12.58","start":"0:02:11.41","end":"0:02:12.58"}]}],"ocr":[{"id":1,"text":"FANDANGO","confidence":0.5057,"left":58,"top":13,"width":74,"height":9,"language":"en-US","instances":[{"adjustedStart":"0:02:24.519","adjustedEnd":"0:02:24.978","start":"0:02:24.519","end":"0:02:24.978"},{"adjustedStart":"0:02:26.813","adjustedEnd":"0:02:27.731","start":"0:02:26.813","end":"0:02:27.731"},{"adjustedStart":"0:02:29.107","adjustedEnd":"0:02:30.526","start":"0:02:29.107","end":"0:02:30.526"},{"adjustedStart":"0:02:30.942","adjustedEnd":"0:02:30.984","start":"0:02:30.942","end":"0:02:30.984"},{"adjustedStart":"0:02:31.401","adjustedEnd":"0:02:31.443","start":"0:02:31.401","end":"0:02:31.443"},{"adjustedStart":"0:02:31.86","adjustedEnd":"0:02:31.902","start":"0:02:31.86","end":"0:02:31.902"},{"adjustedStart":"0:02:32.319","adjustedEnd":"0:02:32.361","start":"0:02:32.319","end":"0:02:32.361"}]},{"id":2,"text":"WATCH MORE OF","confidence":0.6162,"left":389,"top":14,"width":142,"height":13,"language":"en-US","instances":[{"adjustedStart":"0:02:23.602","adjustedEnd":"0:02:30.526","start":"0:02:23.602","end":"0:02:30.526"},{"adjustedStart":"0:02:30.942","adjustedEnd":"0:02:30.984","start":"0:02:30.942","end":"0:02:30.984"},{"adjustedStart":"0:02:31.401","adjustedEnd":"0:02:31.443","start":"0:02:31.401","end":"0:02:31.443"},{"adjustedStart":"0:02:31.86","adjustedEnd":"0:02:31.902","start":"0:02:31.86","end":"0:02:31.902"},{"adjustedStart":"0:02:32.319","adjustedEnd":"0:02:32.361","start":"0:02:32.319","end":"0:02:32.361"}]},{"id":3,"text":"MOVIECLIPS","confidence":0.3449,"left":58,"top":23,"width":139,"height":18,"language":"en-US","instances":[{"adjustedStart":"0:02:26.355","adjustedEnd":"0:02:28.649","start":"0:02:26.355","end":"0:02:28.649"},{"adjustedStart":"0:02:29.107","adjustedEnd":"0:02:30.484","start":"0:02:29.107","end":"0:02:30.484"},{"adjustedStart":"0:02:30.942","adjustedEnd":"0:02:30.984","start":"0:02:30.942","end":"0:02:30.984"},{"adjustedStart":"0:02:31.401","adjustedEnd":"0:02:31.443","start":"0:02:31.401","end":"0:02:31.443"},{"adjustedStart":"0:02:31.86","adjustedEnd":"0:02:31.902","start":"0:02:31.86","end":"0:02:31.902"},{"adjustedStart":"0:02:32.319","adjustedEnd":"0:02:32.361","start":"0:02:32.319","end":"0:02:32.361"}]},{"id":4,"text":"CLIPS","confidence":0.295,"left":133,"top":23,"width":64,"height":18,"language":"en-US","instances":[{"adjustedStart":"0:02:24.978","adjustedEnd":"0:02:25.437","start":"0:02:24.978","end":"0:02:25.437"}]},{"id":5,"text":"YOUR","confidence":0.743,"left":389,"top":30,"width":47,"height":12,"language":"en-US","instances":[{"adjustedStart":"0:02:23.602","adjustedEnd":"0:02:24.061","start":"0:02:23.602","end":"0:02:24.061"}]},{"id":6,"text":"YOUR FAVORITE CLIPS","confidence":0.5199,"left":389,"top":30,"width":185,"height":12,"language":"en-US","instances":[{"adjustedStart":"0:02:24.061","adjustedEnd":"0:02:30.526","start":"0:02:24.061","end":"0:02:30.526"},{"adjustedStart":"0:02:30.942","adjustedEnd":"0:02:30.984","start":"0:02:30.942","end":"0:02:30.984"},{"adjustedStart":"0:02:31.401","adjustedEnd":"0:02:31.443","start":"0:02:31.401","end":"0:02:31.443"},{"adjustedStart":"0:02:31.86","adjustedEnd":"0:02:31.902","start":"0:02:31.86","end":"0:02:31.902"},{"adjustedStart":"0:02:32.319","adjustedEnd":"0:02:32.361","start":"0:02:32.319","end":"0:02:32.361"}]},{"id":7,"text":"SUBSCRIBE","confidence":0.4938,"left":205,"top":93,"width":121,"height":17,"language":"en-US","instances":[{"adjustedStart":"0:02:23.143","adjustedEnd":"0:02:30.526","start":"0:02:23.143","end":"0:02:30.526"},{"adjustedStart":"0:02:30.942","adjustedEnd":"0:02:30.984","start":"0:02:30.942","end":"0:02:30.984"},{"adjustedStart":"0:02:31.401","adjustedEnd":"0:02:31.443","start":"0:02:31.401","end":"0:02:31.443"},{"adjustedStart":"0:02:31.86","adjustedEnd":"0:02:31.902","start":"0:02:31.86","end":"0:02:31.902"},{"adjustedStart":"0:02:32.319","adjustedEnd":"0:02:32.361","start":"0:02:32.319","end":"0:02:32.361"}]},{"id":8,"text":"HERE","confidence":0.445,"left":271,"top":113,"width":55,"height":15,"language":"en-US","instances":[{"adjustedStart":"0:02:23.143","adjustedEnd":"0:02:30.526","start":"0:02:23.143","end":"0:02:30.526"},{"adjustedStart":"0:02:30.942","adjustedEnd":"0:02:30.984","start":"0:02:30.942","end":"0:02:30.984"},{"adjustedStart":"0:02:31.401","adjustedEnd":"0:02:31.443","start":"0:02:31.401","end":"0:02:31.443"},{"adjustedStart":"0:02:31.86","adjustedEnd":"0:02:31.902","start":"0:02:31.86","end":"0:02:31.902"},{"adjustedStart":"0:02:32.319","adjustedEnd":"0:02:32.361","start":"0:02:32.319","end":"0:02:32.361"}]},{"id":9,"text":"BUY OR RENT","confidence":0.55,"left":183,"top":245,"width":144,"height":16,"language":"en-US","instances":[{"adjustedStart":"0:02:23.143","adjustedEnd":"0:02:30.526","start":"0:02:23.143","end":"0:02:30.526"},{"adjustedStart":"0:02:30.942","adjustedEnd":"0:02:30.984","start":"0:02:30.942","end":"0:02:30.984"},{"adjustedStart":"0:02:31.401","adjustedEnd":"0:02:31.443","start":"0:02:31.401","end":"0:02:31.443"},{"adjustedStart":"0:02:31.86","adjustedEnd":"0:02:31.902","start":"0:02:31.86","end":"0:02:31.902"},{"adjustedStart":"0:02:32.319","adjustedEnd":"0:02:32.361","start":"0:02:32.319","end":"0:02:32.361"}]},{"id":10,"text":"THE FULL MOVIE","confidence":0.4715,"left":151,"top":264,"width":175,"height":16,"language":"en-US","instances":[{"adjustedStart":"0:02:23.143","adjustedEnd":"0:02:30.526","start":"0:02:23.143","end":"0:02:30.526"},{"adjustedStart":"0:02:30.942","adjustedEnd":"0:02:30.984","start":"0:02:30.942","end":"0:02:30.984"},{"adjustedStart":"0:02:31.401","adjustedEnd":"0:02:31.443","start":"0:02:31.401","end":"0:02:31.443"},{"adjustedStart":"0:02:31.86","adjustedEnd":"0:02:31.902","start":"0:02:31.86","end":"0:02:31.902"},{"adjustedStart":"0:02:32.319","adjustedEnd":"0:02:32.361","start":"0:02:32.319","end":"0:02:32.361"}]},{"id":11,"text":"MOVIECLIPS","confidence":0.3,"left":39,"top":303,"width":87,"height":24,"language":"en-US","instances":[{"adjustedStart":"0:02:01.121","adjustedEnd":"0:02:01.58","start":"0:02:01.121","end":"0:02:01.58"}]},{"id":12,"text":"MOVIECLIPS","confidence":0.3618,"left":51,"top":331,"width":87,"height":11,"language":"en-US","instances":[{"adjustedStart":"0:00:08.258","adjustedEnd":"0:00:08.717","start":"0:00:08.258","end":"0:00:08.717"},{"adjustedStart":"0:00:30.28","adjustedEnd":"0:00:30.739","start":"0:00:30.28","end":"0:00:30.739"},{"adjustedStart":"0:00:51.843","adjustedEnd":"0:00:55.055","start":"0:00:51.843","end":"0:00:55.055"},{"adjustedStart":"0:01:01.937","adjustedEnd":"0:01:02.396","start":"0:01:01.937","end":"0:01:02.396"},{"adjustedStart":"0:01:03.313","adjustedEnd":"0:01:07.313","start":"0:01:03.313","end":"0:01:07.313"},{"adjustedStart":"0:01:27.17","adjustedEnd":"0:01:27.629","start":"0:01:27.17","end":"0:01:27.629"},{"adjustedStart":"0:01:28.088","adjustedEnd":"0:01:29.923","start":"0:01:28.088","end":"0:01:29.923"},{"adjustedStart":"0:02:13.967","adjustedEnd":"0:02:14.426","start":"0:02:13.967","end":"0:02:14.426"},{"adjustedStart":"0:02:19.931","adjustedEnd":"0:02:20.39","start":"0:02:19.931","end":"0:02:20.39"}]},{"id":13,"text":"ECLIPS","confidence":0.267,"left":96,"top":344,"width":54,"height":14,"language":"en-US","instances":[{"adjustedStart":"0:00:15.599","adjustedEnd":"0:00:17.893","start":"0:00:15.599","end":"0:00:17.893"}]},{"id":14,"text":"IECUPS","confidence":0.341,"left":96,"top":344,"width":53,"height":14,"language":"en-US","instances":[{"adjustedStart":"0:00:37.621","adjustedEnd":"0:00:38.08","start":"0:00:37.621","end":"0:00:38.08"}]}],"keywords":[{"id":1,"text":"clips","confidence":0.8041,"language":"en-US","instances":[{"adjustedStart":"0:02:26.355","adjustedEnd":"0:02:28.649","start":"0:02:26.355","end":"0:02:28.649"},{"adjustedStart":"0:02:24.978","adjustedEnd":"0:02:25.437","start":"0:02:24.978","end":"0:02:25.437"},{"adjustedStart":"0:02:24.061","adjustedEnd":"0:02:30.526","start":"0:02:24.061","end":"0:02:30.526"},{"adjustedStart":"0:02:01.121","adjustedEnd":"0:02:01.58","start":"0:02:01.121","end":"0:02:01.58"},{"adjustedStart":"0:00:08.258","adjustedEnd":"0:00:08.717","start":"0:00:08.258","end":"0:00:08.717"},{"adjustedStart":"0:00:15.599","adjustedEnd":"0:00:17.893","start":"0:00:15.599","end":"0:00:17.893"}]},{"id":2,"text":"monica","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:01:08.73","adjustedEnd":"0:01:16.92","start":"0:01:08.73","end":"0:01:16.92"},{"adjustedStart":"0:01:20.94","adjustedEnd":"0:01:24.84","start":"0:01:20.94","end":"0:01:24.84"},{"adjustedStart":"0:01:24.84","adjustedEnd":"0:01:27.33","start":"0:01:24.84","end":"0:01:27.33"}]},{"id":3,"text":"back","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:01:34.69","adjustedEnd":"0:01:38.06","start":"0:01:34.69","end":"0:01:38.06"},{"adjustedStart":"0:01:39.46","adjustedEnd":"0:01:41.43","start":"0:01:39.46","end":"0:01:41.43"}]},{"id":4,"text":"movieclips","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:02:26.355","adjustedEnd":"0:02:28.649","start":"0:02:26.355","end":"0:02:28.649"},{"adjustedStart":"0:02:01.121","adjustedEnd":"0:02:01.58","start":"0:02:01.121","end":"0:02:01.58"},{"adjustedStart":"0:00:08.258","adjustedEnd":"0:00:08.717","start":"0:00:08.258","end":"0:00:08.717"}]},{"id":5,"text":"eclips","confidence":0.6211,"language":"en-US","instances":[{"adjustedStart":"0:02:26.355","adjustedEnd":"0:02:28.649","start":"0:02:26.355","end":"0:02:28.649"},{"adjustedStart":"0:02:01.121","adjustedEnd":"0:02:01.58","start":"0:02:01.121","end":"0:02:01.58"},{"adjustedStart":"0:00:08.258","adjustedEnd":"0:00:08.717","start":"0:00:08.258","end":"0:00:08.717"},{"adjustedStart":"0:00:15.599","adjustedEnd":"0:00:17.893","start":"0:00:15.599","end":"0:00:17.893"}]}],"faces":[{"id":1049,"name":"Sandra Bullock","confidence":0.8664,"description":"Sandra Annette Bullock is an American-German actress, producer, and philanthropist. She was the highest paid actress in the world in 2010 and 2014. In 2015, Bullock was chosen as People's Most Beautiful Woman and was included in Time's 100 most influential people in the world in 2010. Bullock is the …","thumbnailId":"d9cc852e-f5ba-4238-921b-0a00865b6841","referenceId":"02e07452-35e9-a82c-f32e-db9dadf6a616","referenceType":"Bing","title":"American Actress","imageUrl":"https://www.bing.com/th?id=AMMS_f2e59c9fd2fed50c4d1fd78a261012c7&c=12&rs=1&qlt=80&cdv=1&pid=16.2","thumbnails":[{"id":"e53a97c8-5552-43b4-9e31-5f0fca0a49e2","fileName":"FaceInstanceThumbnail_e53a97c8-5552-43b4-9e31-5f0fca0a49e2.jpg","instances":[{"adjustedStart":"0:00:05.589","adjustedEnd":"0:00:05.631","start":"0:00:05.589","end":"0:00:05.631"}]},{"id":"e8ca0fdd-9f67-44c5-bdf9-dc98e361b060","fileName":"FaceInstanceThumbnail_e8ca0fdd-9f67-44c5-bdf9-dc98e361b060.jpg","instances":[{"adjustedStart":"0:00:15.015","adjustedEnd":"0:00:15.057","start":"0:00:15.015","end":"0:00:15.057"}]},{"id":"36adf039-ddf2-4f3d-b109-aec49f11d97b","fileName":"FaceInstanceThumbnail_36adf039-ddf2-4f3d-b109-aec49f11d97b.jpg","instances":[{"adjustedStart":"0:00:18.352","adjustedEnd":"0:00:18.394","start":"0:00:18.352","end":"0:00:18.394"}]},{"id":"b03454eb-1d3f-4e8d-af5d-aef67ee88dce","fileName":"FaceInstanceThumbnail_b03454eb-1d3f-4e8d-af5d-aef67ee88dce.jpg","instances":[{"adjustedStart":"0:00:20.604","adjustedEnd":"0:00:20.646","start":"0:00:20.604","end":"0:00:20.646"}]},{"id":"674bd07d-a74d-42fa-9397-a9b7d1f37ec4","fileName":"FaceInstanceThumbnail_674bd07d-a74d-42fa-9397-a9b7d1f37ec4.jpg","instances":[{"adjustedStart":"0:00:32.616","adjustedEnd":"0:00:32.658","start":"0:00:32.616","end":"0:00:32.658"}]},{"id":"36fc8000-39f3-4e36-8d2b-f735cd6ca3a4","fileName":"FaceInstanceThumbnail_36fc8000-39f3-4e36-8d2b-f735cd6ca3a4.jpg","instances":[{"adjustedStart":"0:00:37.621","adjustedEnd":"0:00:37.663","start":"0:00:37.621","end":"0:00:37.663"}]},{"id":"c2c0a998-330b-4bba-a30b-4b189b242f53","fileName":"FaceInstanceThumbnail_c2c0a998-330b-4bba-a30b-4b189b242f53.jpg","instances":[{"adjustedStart":"0:00:42.626","adjustedEnd":"0:00:42.668","start":"0:00:42.626","end":"0:00:42.668"}]},{"id":"20486a4a-c63b-403f-a886-048ed5d09d65","fileName":"FaceInstanceThumbnail_20486a4a-c63b-403f-a886-048ed5d09d65.jpg","instances":[{"adjustedStart":"0:00:53.804","adjustedEnd":"0:00:53.846","start":"0:00:53.804","end":"0:00:53.846"}]},{"id":"54f95063-de82-4c4a-aff8-3909886442e9","fileName":"FaceInstanceThumbnail_54f95063-de82-4c4a-aff8-3909886442e9.jpg","instances":[{"adjustedStart":"0:01:24.585","adjustedEnd":"0:01:24.627","start":"0:01:24.585","end":"0:01:24.627"}]},{"id":"d9cc852e-f5ba-4238-921b-0a00865b6841","fileName":"FaceInstanceThumbnail_d9cc852e-f5ba-4238-921b-0a00865b6841.jpg","instances":[{"adjustedStart":"0:01:29.59","adjustedEnd":"0:01:29.632","start":"0:01:29.59","end":"0:01:29.632"}]},{"id":"f2bb4b8e-de2c-43c8-acce-acd82a07b942","fileName":"FaceInstanceThumbnail_f2bb4b8e-de2c-43c8-acce-acd82a07b942.jpg","instances":[{"adjustedStart":"0:01:34.595","adjustedEnd":"0:01:34.637","start":"0:01:34.595","end":"0:01:34.637"}]},{"id":"a95d3a80-36b8-42a4-b737-f3ad2dfdf197","fileName":"FaceInstanceThumbnail_a95d3a80-36b8-42a4-b737-f3ad2dfdf197.jpg","instances":[{"adjustedStart":"0:01:39.6","adjustedEnd":"0:01:39.642","start":"0:01:39.6","end":"0:01:39.642"}]},{"id":"5ea65a3d-a17a-441f-a5b1-37ec4f33f683","fileName":"FaceInstanceThumbnail_5ea65a3d-a17a-441f-a5b1-37ec4f33f683.jpg","instances":[{"adjustedStart":"0:01:50.527","adjustedEnd":"0:01:50.569","start":"0:01:50.527","end":"0:01:50.569"}]},{"id":"159ca77f-d282-4ec8-853c-9f7cfa2105fb","fileName":"FaceInstanceThumbnail_159ca77f-d282-4ec8-853c-9f7cfa2105fb.jpg","instances":[{"adjustedStart":"0:01:53.28","adjustedEnd":"0:01:53.322","start":"0:01:53.28","end":"0:01:53.322"}]},{"id":"b15bb037-c969-4e3c-8bd7-3a901133c54f","fileName":"FaceInstanceThumbnail_b15bb037-c969-4e3c-8bd7-3a901133c54f.jpg","instances":[{"adjustedStart":"0:02:14.468","adjustedEnd":"0:02:14.51","start":"0:02:14.468","end":"0:02:14.51"}]}],"instances":[{"thumbnailsIds":["e53a97c8-5552-43b4-9e31-5f0fca0a49e2"],"adjustedStart":"0:00:05.589","adjustedEnd":"0:00:10.135","start":"0:00:05.589","end":"0:00:10.135"},{"thumbnailsIds":["e8ca0fdd-9f67-44c5-bdf9-dc98e361b060","36adf039-ddf2-4f3d-b109-aec49f11d97b"],"adjustedStart":"0:00:15.015","adjustedEnd":"0:00:18.852","start":"0:00:15.015","end":"0:00:18.852"},{"thumbnailsIds":["b03454eb-1d3f-4e8d-af5d-aef67ee88dce"],"adjustedStart":"0:00:20.604","adjustedEnd":"0:00:21.564","start":"0:00:20.604","end":"0:00:21.564"},{"adjustedStart":"0:00:22.189","adjustedEnd":"0:00:22.273","start":"0:00:22.189","end":"0:00:22.273"},{"adjustedStart":"0:00:24.191","adjustedEnd":"0:00:24.9","start":"0:00:24.191","end":"0:00:24.9"},{"adjustedStart":"0:00:26.527","adjustedEnd":"0:00:27.152","start":"0:00:26.527","end":"0:00:27.152"},{"thumbnailsIds":["674bd07d-a74d-42fa-9397-a9b7d1f37ec4"],"adjustedStart":"0:00:29.613","adjustedEnd":"0:00:34.702","start":"0:00:29.613","end":"0:00:34.702"},{"thumbnailsIds":["36fc8000-39f3-4e36-8d2b-f735cd6ca3a4"],"adjustedStart":"0:00:36.119","adjustedEnd":"0:00:38.831","start":"0:00:36.119","end":"0:00:38.831"},{"thumbnailsIds":["c2c0a998-330b-4bba-a30b-4b189b242f53"],"adjustedStart":"0:00:40.707","adjustedEnd":"0:00:43.335","start":"0:00:40.707","end":"0:00:43.335"},{"adjustedStart":"0:00:45.045","adjustedEnd":"0:00:47.423","start":"0:00:45.045","end":"0:00:47.423"},{"thumbnailsIds":["20486a4a-c63b-403f-a886-048ed5d09d65"],"adjustedStart":"0:00:49.049","adjustedEnd":"0:00:56.181","start":"0:00:49.049","end":"0:00:56.181"},{"thumbnailsIds":["54f95063-de82-4c4a-aff8-3909886442e9","d9cc852e-f5ba-4238-921b-0a00865b6841","f2bb4b8e-de2c-43c8-acce-acd82a07b942","a95d3a80-36b8-42a4-b737-f3ad2dfdf197"],"adjustedStart":"0:01:24.585","adjustedEnd":"0:01:40.934","start":"0:01:24.585","end":"0:01:40.934"},{"thumbnailsIds":["5ea65a3d-a17a-441f-a5b1-37ec4f33f683","159ca77f-d282-4ec8-853c-9f7cfa2105fb"],"adjustedStart":"0:01:50.527","adjustedEnd":"0:01:54.323","start":"0:01:50.527","end":"0:01:54.323"},{"thumbnailsIds":["b15bb037-c969-4e3c-8bd7-3a901133c54f"],"adjustedStart":"0:02:14.468","adjustedEnd":"0:02:16.011","start":"0:02:14.468","end":"0:02:16.011"},{"adjustedStart":"0:02:16.553","adjustedEnd":"0:02:18.931","start":"0:02:16.553","end":"0:02:18.931"}]},{"id":1621,"name":"Unknown #1","confidence":0,"description":null,"thumbnailId":"d008f274-463a-422f-a2f5-fa4708d33533","title":null,"imageUrl":null,"thumbnails":[{"id":"40496323-eaf0-4fa7-999f-2bc2d6366bee","fileName":"FaceInstanceThumbnail_40496323-eaf0-4fa7-999f-2bc2d6366bee.jpg","instances":[{"adjustedStart":"0:01:55.616","adjustedEnd":"0:01:55.658","start":"0:01:55.616","end":"0:01:55.658"}]},{"id":"d008f274-463a-422f-a2f5-fa4708d33533","fileName":"FaceInstanceThumbnail_d008f274-463a-422f-a2f5-fa4708d33533.jpg","instances":[{"adjustedStart":"0:02:00.621","adjustedEnd":"0:02:00.663","start":"0:02:00.621","end":"0:02:00.663"}]},{"id":"e1153616-ab86-4e47-9fe3-56c7c497e613","fileName":"FaceInstanceThumbnail_e1153616-ab86-4e47-9fe3-56c7c497e613.jpg","instances":[{"adjustedStart":"0:02:21.141","adjustedEnd":"0:02:21.183","start":"0:02:21.141","end":"0:02:21.183"}]}],"instances":[{"thumbnailsIds":["40496323-eaf0-4fa7-999f-2bc2d6366bee","d008f274-463a-422f-a2f5-fa4708d33533"],"adjustedStart":"0:01:55.616","adjustedEnd":"0:02:05.25","start":"0:01:55.616","end":"0:02:05.25"},{"thumbnailsIds":["e1153616-ab86-4e47-9fe3-56c7c497e613"],"adjustedStart":"0:02:21.141","adjustedEnd":"0:02:22.434","start":"0:02:21.141","end":"0:02:22.434"}]},{"id":1308,"name":"Unknown #2","confidence":0,"description":null,"thumbnailId":"9f079bb6-b60b-4431-baa7-ca0bfa993314","title":null,"imageUrl":null,"thumbnails":[{"id":"9446dce8-ed7c-4192-8a64-6be87bf1ad96","fileName":"FaceInstanceThumbnail_9446dce8-ed7c-4192-8a64-6be87bf1ad96.jpg","instances":[{"adjustedStart":"0:00:47.631","adjustedEnd":"0:00:47.673","start":"0:00:47.631","end":"0:00:47.673"}]},{"id":"9f079bb6-b60b-4431-baa7-ca0bfa993314","fileName":"FaceInstanceThumbnail_9f079bb6-b60b-4431-baa7-ca0bfa993314.jpg","instances":[{"adjustedStart":"0:00:48.799","adjustedEnd":"0:00:48.841","start":"0:00:48.799","end":"0:00:48.841"}]}],"instances":[{"adjustedStart":"0:00:34.701","adjustedEnd":"0:00:36.12","start":"0:00:34.701","end":"0:00:36.12"},{"adjustedStart":"0:00:38.872","adjustedEnd":"0:00:40.708","start":"0:00:38.872","end":"0:00:40.708"},{"adjustedStart":"0:00:43.377","adjustedEnd":"0:00:47.089","start":"0:00:43.377","end":"0:00:47.089"},{"thumbnailsIds":["9446dce8-ed7c-4192-8a64-6be87bf1ad96","9f079bb6-b60b-4431-baa7-ca0bfa993314"],"adjustedStart":"0:00:47.464","adjustedEnd":"0:00:48.924","start":"0:00:47.464","end":"0:00:48.924"}]},{"id":1083,"name":"Unknown #4","confidence":0,"description":null,"thumbnailId":"60831835-6654-4805-a254-f239db6f4dc1","title":null,"imageUrl":null,"thumbnails":[{"id":"60831835-6654-4805-a254-f239db6f4dc1","fileName":"FaceInstanceThumbnail_60831835-6654-4805-a254-f239db6f4dc1.jpg","instances":[{"adjustedStart":"0:00:24.942","adjustedEnd":"0:00:24.984","start":"0:00:24.942","end":"0:00:24.984"}]}],"instances":[{"thumbnailsIds":["60831835-6654-4805-a254-f239db6f4dc1"],"adjustedStart":"0:00:24.942","adjustedEnd":"0:00:27.069","start":"0:00:24.942","end":"0:00:27.069"}]},{"id":1089,"name":"Unknown #6","confidence":0,"description":null,"thumbnailId":"c7c0b07e-d760-43c3-9d5c-8b3413883e58","title":null,"imageUrl":null,"thumbnails":[{"id":"fc742d0b-fc99-4ef2-b37c-3f59df56311a","fileName":"FaceInstanceThumbnail_fc742d0b-fc99-4ef2-b37c-3f59df56311a.jpg","instances":[{"adjustedStart":"0:00:27.194","adjustedEnd":"0:00:27.236","start":"0:00:27.194","end":"0:00:27.236"}]},{"id":"c7c0b07e-d760-43c3-9d5c-8b3413883e58","fileName":"FaceInstanceThumbnail_c7c0b07e-d760-43c3-9d5c-8b3413883e58.jpg","instances":[{"adjustedStart":"0:00:28.362","adjustedEnd":"0:00:28.404","start":"0:00:28.362","end":"0:00:28.404"}]}],"instances":[{"thumbnailsIds":["fc742d0b-fc99-4ef2-b37c-3f59df56311a","c7c0b07e-d760-43c3-9d5c-8b3413883e58"],"adjustedStart":"0:00:27.194","adjustedEnd":"0:00:28.404","start":"0:00:27.194","end":"0:00:28.404"}]}],"labels":[{"id":1,"name":"screenshot","language":"en-US","instances":[{"confidence":0.9736,"adjustedStart":"0:00:00","adjustedEnd":"0:00:02.67","start":"0:00:00","end":"0:00:02.67"},{"confidence":0.9509,"adjustedStart":"0:00:05.339","adjustedEnd":"0:00:10.678","start":"0:00:05.339","end":"0:00:10.678"},{"confidence":0.9661,"adjustedStart":"0:00:16.016","adjustedEnd":"0:00:25.359","start":"0:00:16.016","end":"0:00:25.359"},{"confidence":0.9049,"adjustedStart":"0:00:28.028","adjustedEnd":"0:00:29.363","start":"0:00:28.028","end":"0:00:29.363"},{"confidence":0.9273,"adjustedStart":"0:00:37.371","adjustedEnd":"0:00:40.04","start":"0:00:37.371","end":"0:00:40.04"},{"confidence":0.9537,"adjustedStart":"0:00:41.375","adjustedEnd":"0:00:48.048","start":"0:00:41.375","end":"0:00:48.048"},{"confidence":0.9529,"adjustedStart":"0:00:57.391","adjustedEnd":"0:01:04.064","start":"0:00:57.391","end":"0:01:04.064"},{"confidence":0.9615,"adjustedStart":"0:01:12.072","adjustedEnd":"0:01:13.407","start":"0:01:12.072","end":"0:01:13.407"},{"confidence":0.9499,"adjustedStart":"0:01:21.415","adjustedEnd":"0:01:22.75","start":"0:01:21.415","end":"0:01:22.75"},{"confidence":0.9552,"adjustedStart":"0:01:24.084","adjustedEnd":"0:01:32.092","start":"0:01:24.084","end":"0:01:32.092"},{"confidence":0.9246,"adjustedStart":"0:01:33.427","adjustedEnd":"0:01:38.766","start":"0:01:33.427","end":"0:01:38.766"},{"confidence":0.9274,"adjustedStart":"0:01:45.439","adjustedEnd":"0:01:50.778","start":"0:01:45.439","end":"0:01:50.778"},{"confidence":0.9232,"adjustedStart":"0:01:53.447","adjustedEnd":"0:01:54.782","start":"0:01:53.447","end":"0:01:54.782"},{"confidence":0.9484,"adjustedStart":"0:02:01.455","adjustedEnd":"0:02:05.459","start":"0:02:01.455","end":"0:02:05.459"},{"confidence":0.9234,"adjustedStart":"0:02:08.128","adjustedEnd":"0:02:09.463","start":"0:02:08.128","end":"0:02:09.463"},{"confidence":0.9082,"adjustedStart":"0:02:10.797","adjustedEnd":"0:02:12.132","start":"0:02:10.797","end":"0:02:12.132"},{"confidence":0.9846,"adjustedStart":"0:02:22.809","adjustedEnd":"0:02:32.444","start":"0:02:22.809","end":"0:02:32.444"}]},{"id":2,"name":"wall","referenceId":"structure/wall","language":"en-US","instances":[{"confidence":0.895,"adjustedStart":"0:00:00","adjustedEnd":"0:00:01.335","start":"0:00:00","end":"0:00:01.335"},{"confidence":0.9311,"adjustedStart":"0:00:06.673","adjustedEnd":"0:00:10.678","start":"0:00:06.673","end":"0:00:10.678"},{"confidence":0.9204,"adjustedStart":"0:00:14.681","adjustedEnd":"0:00:21.355","start":"0:00:14.681","end":"0:00:21.355"},{"confidence":0.9453,"adjustedStart":"0:00:25.359","adjustedEnd":"0:00:44.044","start":"0:00:25.359","end":"0:00:44.044"},{"confidence":0.9625,"adjustedStart":"0:00:45.379","adjustedEnd":"0:00:57.391","start":"0:00:45.379","end":"0:00:57.391"},{"confidence":0.9416,"adjustedStart":"0:01:02.729","adjustedEnd":"0:01:16.076","start":"0:01:02.729","end":"0:01:16.076"},{"confidence":0.932,"adjustedStart":"0:01:41.435","adjustedEnd":"0:01:42.77","start":"0:01:41.435","end":"0:01:42.77"},{"confidence":0.9362,"adjustedStart":"0:01:44.104","adjustedEnd":"0:01:48.108","start":"0:01:44.104","end":"0:01:48.108"},{"confidence":0.9454,"adjustedStart":"0:01:49.443","adjustedEnd":"0:01:58.786","start":"0:01:49.443","end":"0:01:58.786"},{"confidence":0.9275,"adjustedStart":"0:02:00.12","adjustedEnd":"0:02:04.124","start":"0:02:00.12","end":"0:02:04.124"},{"confidence":0.966,"adjustedStart":"0:02:05.459","adjustedEnd":"0:02:16.136","start":"0:02:05.459","end":"0:02:16.136"},{"confidence":0.9568,"adjustedStart":"0:02:17.471","adjustedEnd":"0:02:22.81","start":"0:02:17.471","end":"0:02:22.81"}]},{"id":3,"name":"keyboard","referenceId":"device/keyboard","language":"en-US","instances":[{"confidence":0.8999,"adjustedStart":"0:00:02.669","adjustedEnd":"0:00:05.339","start":"0:00:02.669","end":"0:00:05.339"}]},{"id":4,"name":"close","language":"en-US","instances":[{"confidence":0.8442,"adjustedStart":"0:00:02.669","adjustedEnd":"0:00:04.004","start":"0:00:02.669","end":"0:00:04.004"}]},{"id":5,"name":"shelf","referenceId":"furniture/shelf","language":"en-US","instances":[{"confidence":0.8194,"adjustedStart":"0:00:04.004","adjustedEnd":"0:00:05.339","start":"0:00:04.004","end":"0:00:05.339"},{"confidence":0.9032,"adjustedStart":"0:00:12.012","adjustedEnd":"0:00:14.682","start":"0:00:12.012","end":"0:00:14.682"},{"confidence":0.9362,"adjustedStart":"0:01:16.076","adjustedEnd":"0:01:17.411","start":"0:01:16.076","end":"0:01:17.411"}]},{"id":6,"name":"person","referenceId":"person","language":"en-US","instances":[{"confidence":0.9682,"adjustedStart":"0:00:05.339","adjustedEnd":"0:00:12.012","start":"0:00:05.339","end":"0:00:12.012"},{"confidence":0.9815,"adjustedStart":"0:00:14.681","adjustedEnd":"0:00:57.391","start":"0:00:14.681","end":"0:00:57.391"},{"confidence":0.9671,"adjustedStart":"0:01:00.06","adjustedEnd":"0:01:16.076","start":"0:01:00.06","end":"0:01:16.076"},{"confidence":0.9931,"adjustedStart":"0:01:20.08","adjustedEnd":"0:01:41.435","start":"0:01:20.08","end":"0:01:41.435"},{"confidence":0.9797,"adjustedStart":"0:01:48.108","adjustedEnd":"0:01:49.443","start":"0:01:48.108","end":"0:01:49.443"},{"confidence":0.969,"adjustedStart":"0:01:50.777","adjustedEnd":"0:01:54.782","start":"0:01:50.777","end":"0:01:54.782"},{"confidence":0.9522,"adjustedStart":"0:01:56.116","adjustedEnd":"0:01:58.786","start":"0:01:56.116","end":"0:01:58.786"},{"confidence":0.9666,"adjustedStart":"0:02:00.12","adjustedEnd":"0:02:04.124","start":"0:02:00.12","end":"0:02:04.124"},{"confidence":0.9901,"adjustedStart":"0:02:06.793","adjustedEnd":"0:02:14.802","start":"0:02:06.793","end":"0:02:14.802"},{"confidence":0.9753,"adjustedStart":"0:02:17.471","adjustedEnd":"0:02:18.806","start":"0:02:17.471","end":"0:02:18.806"}]},{"id":7,"name":"indoor","language":"en-US","instances":[{"confidence":0.9236,"adjustedStart":"0:00:05.339","adjustedEnd":"0:00:21.355","start":"0:00:05.339","end":"0:00:21.355"},{"confidence":0.9567,"adjustedStart":"0:00:22.689","adjustedEnd":"0:00:57.391","start":"0:00:22.689","end":"0:00:57.391"},{"confidence":0.9054,"adjustedStart":"0:01:00.06","adjustedEnd":"0:01:18.746","start":"0:01:00.06","end":"0:01:18.746"},{"confidence":0.8515,"adjustedStart":"0:01:34.761","adjustedEnd":"0:01:37.431","start":"0:01:34.761","end":"0:01:37.431"},{"confidence":0.9494,"adjustedStart":"0:01:41.435","adjustedEnd":"0:02:16.136","start":"0:01:41.435","end":"0:02:16.136"},{"confidence":0.9624,"adjustedStart":"0:02:17.471","adjustedEnd":"0:02:22.81","start":"0:02:17.471","end":"0:02:22.81"}]},{"id":8,"name":"clothing","referenceId":"consumer goods/clothing","language":"en-US","instances":[{"confidence":0.93,"adjustedStart":"0:00:05.339","adjustedEnd":"0:00:09.343","start":"0:00:05.339","end":"0:00:09.343"},{"confidence":0.9185,"adjustedStart":"0:00:21.355","adjustedEnd":"0:00:22.69","start":"0:00:21.355","end":"0:00:22.69"},{"confidence":0.9722,"adjustedStart":"0:00:24.024","adjustedEnd":"0:00:37.371","start":"0:00:24.024","end":"0:00:37.371"},{"confidence":0.9446,"adjustedStart":"0:00:38.705","adjustedEnd":"0:00:45.379","start":"0:00:38.705","end":"0:00:45.379"},{"confidence":0.9657,"adjustedStart":"0:00:46.713","adjustedEnd":"0:00:57.391","start":"0:00:46.713","end":"0:00:57.391"},{"confidence":0.9516,"adjustedStart":"0:01:01.395","adjustedEnd":"0:01:02.73","start":"0:01:01.395","end":"0:01:02.73"},{"confidence":0.9688,"adjustedStart":"0:01:04.064","adjustedEnd":"0:01:06.734","start":"0:01:04.064","end":"0:01:06.734"},{"confidence":0.9665,"adjustedStart":"0:01:08.068","adjustedEnd":"0:01:14.742","start":"0:01:08.068","end":"0:01:14.742"},{"confidence":0.9282,"adjustedStart":"0:01:20.08","adjustedEnd":"0:01:25.419","start":"0:01:20.08","end":"0:01:25.419"},{"confidence":0.9749,"adjustedStart":"0:01:28.088","adjustedEnd":"0:01:41.435","start":"0:01:28.088","end":"0:01:41.435"},{"confidence":0.9136,"adjustedStart":"0:01:48.108","adjustedEnd":"0:01:50.778","start":"0:01:48.108","end":"0:01:50.778"},{"confidence":0.9288,"adjustedStart":"0:01:52.112","adjustedEnd":"0:01:53.447","start":"0:01:52.112","end":"0:01:53.447"},{"confidence":0.9089,"adjustedStart":"0:02:00.12","adjustedEnd":"0:02:01.455","start":"0:02:00.12","end":"0:02:01.455"},{"confidence":0.9471,"adjustedStart":"0:02:06.793","adjustedEnd":"0:02:14.802","start":"0:02:06.793","end":"0:02:14.802"},{"confidence":0.9642,"adjustedStart":"0:02:17.471","adjustedEnd":"0:02:18.806","start":"0:02:17.471","end":"0:02:18.806"}]},{"id":9,"name":"hair","referenceId":"body covering/hair","language":"en-US","instances":[{"confidence":0.9129,"adjustedStart":"0:00:05.339","adjustedEnd":"0:00:10.678","start":"0:00:05.339","end":"0:00:10.678"},{"confidence":0.9121,"adjustedStart":"0:00:14.681","adjustedEnd":"0:00:21.355","start":"0:00:14.681","end":"0:00:21.355"},{"confidence":0.9501,"adjustedStart":"0:00:24.024","adjustedEnd":"0:00:25.359","start":"0:00:24.024","end":"0:00:25.359"},{"confidence":0.9182,"adjustedStart":"0:00:28.028","adjustedEnd":"0:00:29.363","start":"0:00:28.028","end":"0:00:29.363"},{"confidence":0.8998,"adjustedStart":"0:00:34.701","adjustedEnd":"0:00:36.036","start":"0:00:34.701","end":"0:00:36.036"},{"confidence":0.8985,"adjustedStart":"0:00:37.371","adjustedEnd":"0:00:40.04","start":"0:00:37.371","end":"0:00:40.04"},{"confidence":0.8953,"adjustedStart":"0:00:41.375","adjustedEnd":"0:00:49.383","start":"0:00:41.375","end":"0:00:49.383"},{"confidence":0.9641,"adjustedStart":"0:01:25.419","adjustedEnd":"0:01:28.088","start":"0:01:25.419","end":"0:01:28.088"},{"confidence":0.9125,"adjustedStart":"0:01:46.773","adjustedEnd":"0:01:48.108","start":"0:01:46.773","end":"0:01:48.108"},{"confidence":0.8977,"adjustedStart":"0:01:49.443","adjustedEnd":"0:01:54.782","start":"0:01:49.443","end":"0:01:54.782"},{"confidence":0.9085,"adjustedStart":"0:01:58.785","adjustedEnd":"0:02:02.79","start":"0:01:58.785","end":"0:02:02.79"},{"confidence":0.8645,"adjustedStart":"0:02:04.124","adjustedEnd":"0:02:05.459","start":"0:02:04.124","end":"0:02:05.459"},{"confidence":0.8708,"adjustedStart":"0:02:18.805","adjustedEnd":"0:02:20.14","start":"0:02:18.805","end":"0:02:20.14"}]},{"id":10,"name":"woman","referenceId":"person/woman","language":"en-US","instances":[{"confidence":0.942,"adjustedStart":"0:00:05.339","adjustedEnd":"0:00:10.678","start":"0:00:05.339","end":"0:00:10.678"},{"confidence":0.9195,"adjustedStart":"0:00:14.681","adjustedEnd":"0:00:29.363","start":"0:00:14.681","end":"0:00:29.363"},{"confidence":0.9407,"adjustedStart":"0:00:30.697","adjustedEnd":"0:00:57.391","start":"0:00:30.697","end":"0:00:57.391"},{"confidence":0.9136,"adjustedStart":"0:01:01.395","adjustedEnd":"0:01:02.73","start":"0:01:01.395","end":"0:01:02.73"},{"confidence":0.9266,"adjustedStart":"0:01:04.064","adjustedEnd":"0:01:10.738","start":"0:01:04.064","end":"0:01:10.738"},{"confidence":0.8623,"adjustedStart":"0:01:17.411","adjustedEnd":"0:01:18.746","start":"0:01:17.411","end":"0:01:18.746"},{"confidence":0.9228,"adjustedStart":"0:01:25.419","adjustedEnd":"0:01:41.435","start":"0:01:25.419","end":"0:01:41.435"},{"confidence":0.8608,"adjustedStart":"0:01:45.439","adjustedEnd":"0:01:46.774","start":"0:01:45.439","end":"0:01:46.774"},{"confidence":0.9345,"adjustedStart":"0:01:50.777","adjustedEnd":"0:01:54.782","start":"0:01:50.777","end":"0:01:54.782"},{"confidence":0.9449,"adjustedStart":"0:01:57.451","adjustedEnd":"0:02:04.124","start":"0:01:57.451","end":"0:02:04.124"},{"confidence":0.9414,"adjustedStart":"0:02:06.793","adjustedEnd":"0:02:16.136","start":"0:02:06.793","end":"0:02:16.136"},{"confidence":0.9313,"adjustedStart":"0:02:17.471","adjustedEnd":"0:02:20.14","start":"0:02:17.471","end":"0:02:20.14"}]},{"id":11,"name":"human face","language":"en-US","instances":[{"confidence":0.9651,"adjustedStart":"0:00:05.339","adjustedEnd":"0:00:10.678","start":"0:00:05.339","end":"0:00:10.678"},{"confidence":0.9519,"adjustedStart":"0:00:14.681","adjustedEnd":"0:00:22.69","start":"0:00:14.681","end":"0:00:22.69"},{"confidence":0.9717,"adjustedStart":"0:00:24.024","adjustedEnd":"0:00:29.363","start":"0:00:24.024","end":"0:00:29.363"},{"confidence":0.9799,"adjustedStart":"0:00:30.697","adjustedEnd":"0:00:52.052","start":"0:00:30.697","end":"0:00:52.052"},{"confidence":0.9776,"adjustedStart":"0:00:54.721","adjustedEnd":"0:00:57.391","start":"0:00:54.721","end":"0:00:57.391"},{"confidence":0.9858,"adjustedStart":"0:01:25.419","adjustedEnd":"0:01:41.435","start":"0:01:25.419","end":"0:01:41.435"},{"confidence":0.9832,"adjustedStart":"0:01:50.777","adjustedEnd":"0:01:54.782","start":"0:01:50.777","end":"0:01:54.782"},{"confidence":0.9711,"adjustedStart":"0:02:00.12","adjustedEnd":"0:02:04.124","start":"0:02:00.12","end":"0:02:04.124"},{"confidence":0.9418,"adjustedStart":"0:02:08.128","adjustedEnd":"0:02:12.132","start":"0:02:08.128","end":"0:02:12.132"},{"confidence":0.9368,"adjustedStart":"0:02:17.471","adjustedEnd":"0:02:18.806","start":"0:02:17.471","end":"0:02:18.806"}]},{"id":12,"name":"girl","referenceId":"person/child/girl","language":"en-US","instances":[{"confidence":0.9718,"adjustedStart":"0:00:06.673","adjustedEnd":"0:00:10.678","start":"0:00:06.673","end":"0:00:10.678"},{"confidence":0.9473,"adjustedStart":"0:00:14.681","adjustedEnd":"0:00:21.355","start":"0:00:14.681","end":"0:00:21.355"},{"confidence":0.9107,"adjustedStart":"0:00:24.024","adjustedEnd":"0:00:25.359","start":"0:00:24.024","end":"0:00:25.359"},{"confidence":0.9111,"adjustedStart":"0:00:26.693","adjustedEnd":"0:00:28.028","start":"0:00:26.693","end":"0:00:28.028"},{"confidence":0.9424,"adjustedStart":"0:00:33.367","adjustedEnd":"0:00:34.702","start":"0:00:33.367","end":"0:00:34.702"},{"confidence":0.9355,"adjustedStart":"0:00:37.371","adjustedEnd":"0:00:40.04","start":"0:00:37.371","end":"0:00:40.04"},{"confidence":0.9449,"adjustedStart":"0:00:41.375","adjustedEnd":"0:00:48.048","start":"0:00:41.375","end":"0:00:48.048"},{"confidence":0.9829,"adjustedStart":"0:01:25.419","adjustedEnd":"0:01:32.092","start":"0:01:25.419","end":"0:01:32.092"},{"confidence":0.9683,"adjustedStart":"0:01:33.427","adjustedEnd":"0:01:41.435","start":"0:01:33.427","end":"0:01:41.435"},{"confidence":0.9928,"adjustedStart":"0:01:45.439","adjustedEnd":"0:01:48.108","start":"0:01:45.439","end":"0:01:48.108"},{"confidence":0.9386,"adjustedStart":"0:01:49.443","adjustedEnd":"0:01:54.782","start":"0:01:49.443","end":"0:01:54.782"},{"confidence":0.9651,"adjustedStart":"0:02:00.12","adjustedEnd":"0:02:04.124","start":"0:02:00.12","end":"0:02:04.124"},{"confidence":0.9922,"adjustedStart":"0:02:08.128","adjustedEnd":"0:02:10.798","start":"0:02:08.128","end":"0:02:10.798"},{"confidence":0.8547,"adjustedStart":"0:02:12.132","adjustedEnd":"0:02:13.467","start":"0:02:12.132","end":"0:02:13.467"},{"confidence":0.9965,"adjustedStart":"0:02:17.471","adjustedEnd":"0:02:18.806","start":"0:02:17.471","end":"0:02:18.806"}]},{"id":13,"name":"long hair","language":"en-US","instances":[{"confidence":0.9292,"adjustedStart":"0:00:08.008","adjustedEnd":"0:00:09.343","start":"0:00:08.008","end":"0:00:09.343"},{"confidence":0.9038,"adjustedStart":"0:00:38.705","adjustedEnd":"0:00:40.04","start":"0:00:38.705","end":"0:00:40.04"}]},{"id":14,"name":"food","referenceId":"food","language":"en-US","instances":[{"confidence":0.9878,"adjustedStart":"0:00:12.012","adjustedEnd":"0:00:13.347","start":"0:00:12.012","end":"0:00:13.347"}]},{"id":15,"name":"bottle","referenceId":"container/bottle","language":"en-US","instances":[{"confidence":0.9005,"adjustedStart":"0:00:12.012","adjustedEnd":"0:00:13.347","start":"0:00:12.012","end":"0:00:13.347"}]},{"id":16,"name":"face","referenceId":"body part/face","language":"en-US","instances":[{"confidence":0.9853,"adjustedStart":"0:00:16.016","adjustedEnd":"0:00:17.351","start":"0:00:16.016","end":"0:00:17.351"},{"confidence":0.9943,"adjustedStart":"0:00:18.685","adjustedEnd":"0:00:20.02","start":"0:00:18.685","end":"0:00:20.02"},{"confidence":0.9913,"adjustedStart":"0:01:25.419","adjustedEnd":"0:01:28.088","start":"0:01:25.419","end":"0:01:28.088"}]},{"id":17,"name":"mirror","referenceId":"mirror","language":"en-US","instances":[{"confidence":0.9492,"adjustedStart":"0:00:20.02","adjustedEnd":"0:00:21.355","start":"0:00:20.02","end":"0:00:21.355"},{"confidence":0.9865,"adjustedStart":"0:00:37.371","adjustedEnd":"0:00:38.706","start":"0:00:37.371","end":"0:00:38.706"},{"confidence":0.9856,"adjustedStart":"0:00:45.379","adjustedEnd":"0:00:46.714","start":"0:00:45.379","end":"0:00:46.714"},{"confidence":0.9865,"adjustedStart":"0:01:02.729","adjustedEnd":"0:01:04.064","start":"0:01:02.729","end":"0:01:04.064"},{"confidence":0.897,"adjustedStart":"0:01:06.733","adjustedEnd":"0:01:08.068","start":"0:01:06.733","end":"0:01:08.068"},{"confidence":0.9634,"adjustedStart":"0:01:44.104","adjustedEnd":"0:01:48.108","start":"0:01:44.104","end":"0:01:48.108"},{"confidence":0.9547,"adjustedStart":"0:01:49.443","adjustedEnd":"0:01:52.112","start":"0:01:49.443","end":"0:01:52.112"},{"confidence":0.9433,"adjustedStart":"0:01:53.447","adjustedEnd":"0:01:54.782","start":"0:01:53.447","end":"0:01:54.782"}]},{"id":18,"name":"text","referenceId":"communication/writing/text","language":"en-US","instances":[{"confidence":0.9901,"adjustedStart":"0:00:25.359","adjustedEnd":"0:00:28.028","start":"0:00:25.359","end":"0:00:28.028"},{"confidence":0.8373,"adjustedStart":"0:00:36.036","adjustedEnd":"0:00:37.371","start":"0:00:36.036","end":"0:00:37.371"},{"confidence":0.9643,"adjustedStart":"0:00:44.044","adjustedEnd":"0:00:45.379","start":"0:00:44.044","end":"0:00:45.379"},{"confidence":0.8579,"adjustedStart":"0:01:18.745","adjustedEnd":"0:01:20.08","start":"0:01:18.745","end":"0:01:20.08"},{"confidence":1,"adjustedStart":"0:02:22.809","adjustedEnd":"0:02:32.444","start":"0:02:22.809","end":"0:02:32.444"}]},{"id":19,"name":"standing","language":"en-US","instances":[{"confidence":0.885,"adjustedStart":"0:00:26.693","adjustedEnd":"0:00:28.028","start":"0:00:26.693","end":"0:00:28.028"},{"confidence":0.9137,"adjustedStart":"0:00:33.367","adjustedEnd":"0:00:34.702","start":"0:00:33.367","end":"0:00:34.702"},{"confidence":0.8917,"adjustedStart":"0:00:49.383","adjustedEnd":"0:00:50.718","start":"0:00:49.383","end":"0:00:50.718"},{"confidence":0.9319,"adjustedStart":"0:01:20.08","adjustedEnd":"0:01:25.419","start":"0:01:20.08","end":"0:01:25.419"},{"confidence":0.9044,"adjustedStart":"0:02:06.793","adjustedEnd":"0:02:09.463","start":"0:02:06.793","end":"0:02:09.463"},{"confidence":0.8956,"adjustedStart":"0:02:13.467","adjustedEnd":"0:02:14.802","start":"0:02:13.467","end":"0:02:14.802"},{"confidence":0.8892,"adjustedStart":"0:02:17.471","adjustedEnd":"0:02:18.806","start":"0:02:17.471","end":"0:02:18.806"}]},{"id":20,"name":"table","referenceId":"furniture/table","language":"en-US","instances":[{"confidence":0.9634,"adjustedStart":"0:00:29.363","adjustedEnd":"0:00:30.698","start":"0:00:29.363","end":"0:00:30.698"},{"confidence":0.9159,"adjustedStart":"0:00:52.052","adjustedEnd":"0:00:53.387","start":"0:00:52.052","end":"0:00:53.387"}]},{"id":21,"name":"furniture","referenceId":"furniture","language":"en-US","instances":[{"confidence":0.9601,"adjustedStart":"0:00:29.363","adjustedEnd":"0:00:30.698","start":"0:00:29.363","end":"0:00:30.698"},{"confidence":0.9434,"adjustedStart":"0:00:52.052","adjustedEnd":"0:00:54.722","start":"0:00:52.052","end":"0:00:54.722"},{"confidence":0.9483,"adjustedStart":"0:01:56.116","adjustedEnd":"0:01:57.451","start":"0:01:56.116","end":"0:01:57.451"}]},{"id":22,"name":"whiteboard","language":"en-US","instances":[{"confidence":0.9158,"adjustedStart":"0:00:29.363","adjustedEnd":"0:00:30.698","start":"0:00:29.363","end":"0:00:30.698"}]},{"id":23,"name":"people","language":"en-US","instances":[{"confidence":0.9242,"adjustedStart":"0:00:32.032","adjustedEnd":"0:00:33.367","start":"0:00:32.032","end":"0:00:33.367"},{"confidence":0.9602,"adjustedStart":"0:01:05.399","adjustedEnd":"0:01:06.734","start":"0:01:05.399","end":"0:01:06.734"}]},{"id":24,"name":"ceiling","referenceId":"surface/ceiling","language":"en-US","instances":[{"confidence":0.9173,"adjustedStart":"0:00:33.367","adjustedEnd":"0:00:34.702","start":"0:00:33.367","end":"0:00:34.702"},{"confidence":0.9006,"adjustedStart":"0:00:54.721","adjustedEnd":"0:00:56.056","start":"0:00:54.721","end":"0:00:56.056"}]},{"id":25,"name":"computer","referenceId":"device/machine/computer","language":"en-US","instances":[{"confidence":0.9361,"adjustedStart":"0:00:34.701","adjustedEnd":"0:00:37.371","start":"0:00:34.701","end":"0:00:37.371"},{"confidence":0.9031,"adjustedStart":"0:00:40.04","adjustedEnd":"0:00:41.375","start":"0:00:40.04","end":"0:00:41.375"}]},{"id":26,"name":"kitchen","referenceId":"structure/room/kitchen","language":"en-US","instances":[{"confidence":0.8572,"adjustedStart":"0:00:34.701","adjustedEnd":"0:00:36.036","start":"0:00:34.701","end":"0:00:36.036"},{"confidence":0.8489,"adjustedStart":"0:00:38.705","adjustedEnd":"0:00:40.04","start":"0:00:38.705","end":"0:00:40.04"}]},{"id":27,"name":"appliance","referenceId":"instrumentality/device/appliance","language":"en-US","instances":[{"confidence":0.8404,"adjustedStart":"0:00:53.387","adjustedEnd":"0:00:54.722","start":"0:00:53.387","end":"0:00:54.722"},{"confidence":0.9036,"adjustedStart":"0:01:57.451","adjustedEnd":"0:01:58.786","start":"0:01:57.451","end":"0:01:58.786"}]},{"id":28,"name":"design","language":"en-US","instances":[{"confidence":0.9873,"adjustedStart":"0:00:54.721","adjustedEnd":"0:00:57.391","start":"0:00:54.721","end":"0:00:57.391"},{"confidence":0.9929,"adjustedStart":"0:02:22.809","adjustedEnd":"0:02:32.444","start":"0:02:22.809","end":"0:02:32.444"}]},{"id":29,"name":"glasses","language":"en-US","instances":[{"confidence":0.9087,"adjustedStart":"0:00:54.721","adjustedEnd":"0:00:56.056","start":"0:00:54.721","end":"0:00:56.056"}]},{"id":30,"name":"art","language":"en-US","instances":[{"confidence":0.994,"adjustedStart":"0:00:57.391","adjustedEnd":"0:01:00.06","start":"0:00:57.391","end":"0:01:00.06"}]},{"id":31,"name":"bag","referenceId":"instrumentality/container/bag","language":"en-US","instances":[{"confidence":0.9091,"adjustedStart":"0:00:58.725","adjustedEnd":"0:01:00.06","start":"0:00:58.725","end":"0:01:00.06"}]},{"id":32,"name":"accessory","referenceId":"covering/clothing/accessory","language":"en-US","instances":[{"confidence":0.8557,"adjustedStart":"0:00:58.725","adjustedEnd":"0:01:00.06","start":"0:00:58.725","end":"0:01:00.06"}]},{"id":33,"name":"video game","language":"en-US","instances":[{"confidence":0.9938,"adjustedStart":"0:01:01.395","adjustedEnd":"0:01:02.73","start":"0:01:01.395","end":"0:01:02.73"},{"confidence":0.9864,"adjustedStart":"0:01:12.072","adjustedEnd":"0:01:13.407","start":"0:01:12.072","end":"0:01:13.407"},{"confidence":0.9954,"adjustedStart":"0:01:20.08","adjustedEnd":"0:01:22.75","start":"0:01:20.08","end":"0:01:22.75"}]},{"id":34,"name":"door","referenceId":"structure/barrier/door","language":"en-US","instances":[{"confidence":0.8032,"adjustedStart":"0:01:04.064","adjustedEnd":"0:01:05.399","start":"0:01:04.064","end":"0:01:05.399"},{"confidence":0.8006,"adjustedStart":"0:01:49.443","adjustedEnd":"0:01:50.778","start":"0:01:49.443","end":"0:01:50.778"},{"confidence":0.8692,"adjustedStart":"0:01:53.447","adjustedEnd":"0:01:54.782","start":"0:01:53.447","end":"0:01:54.782"}]},{"id":35,"name":"room","referenceId":"structure/room","language":"en-US","instances":[{"confidence":0.9219,"adjustedStart":"0:01:05.399","adjustedEnd":"0:01:06.734","start":"0:01:05.399","end":"0:01:06.734"},{"confidence":0.962,"adjustedStart":"0:01:14.741","adjustedEnd":"0:01:16.076","start":"0:01:14.741","end":"0:01:16.076"},{"confidence":0.9316,"adjustedStart":"0:02:05.459","adjustedEnd":"0:02:06.794","start":"0:02:05.459","end":"0:02:06.794"},{"confidence":0.9519,"adjustedStart":"0:02:08.128","adjustedEnd":"0:02:12.132","start":"0:02:08.128","end":"0:02:12.132"},{"confidence":0.8774,"adjustedStart":"0:02:20.14","adjustedEnd":"0:02:21.475","start":"0:02:20.14","end":"0:02:21.475"}]},{"id":36,"name":"flower","referenceId":"plant/flower","language":"en-US","instances":[{"confidence":0.9631,"adjustedStart":"0:01:09.403","adjustedEnd":"0:01:10.738","start":"0:01:09.403","end":"0:01:10.738"}]},{"id":37,"name":"vase","referenceId":"vase","language":"en-US","instances":[{"confidence":0.9096,"adjustedStart":"0:01:09.403","adjustedEnd":"0:01:10.738","start":"0:01:09.403","end":"0:01:10.738"}]},{"id":38,"name":"man","referenceId":"person/man","language":"en-US","instances":[{"confidence":0.9267,"adjustedStart":"0:01:12.072","adjustedEnd":"0:01:14.742","start":"0:01:12.072","end":"0:01:14.742"},{"confidence":0.9346,"adjustedStart":"0:01:20.08","adjustedEnd":"0:01:25.419","start":"0:01:20.08","end":"0:01:25.419"}]},{"id":39,"name":"suit","referenceId":"consumer goods/clothing/suit","language":"en-US","instances":[{"confidence":0.9266,"adjustedStart":"0:01:13.407","adjustedEnd":"0:01:14.742","start":"0:01:13.407","end":"0:01:14.742"},{"confidence":0.9434,"adjustedStart":"0:01:21.415","adjustedEnd":"0:01:25.419","start":"0:01:21.415","end":"0:01:25.419"}]},{"id":40,"name":"display","referenceId":"communication/demonstration/display","language":"en-US","instances":[{"confidence":0.8991,"adjustedStart":"0:01:16.076","adjustedEnd":"0:01:17.411","start":"0:01:16.076","end":"0:01:17.411"},{"confidence":0.9446,"adjustedStart":"0:02:24.144","adjustedEnd":"0:02:32.152","start":"0:02:24.144","end":"0:02:32.152"}]},{"id":41,"name":"car","referenceId":"conveyance/vehicle/car","language":"en-US","instances":[{"confidence":0.959,"adjustedStart":"0:01:20.08","adjustedEnd":"0:01:25.419","start":"0:01:20.08","end":"0:01:25.419"}]},{"id":42,"name":"vehicle","referenceId":"instrumentality/conveyance/vehicle","language":"en-US","instances":[{"confidence":0.9253,"adjustedStart":"0:01:20.08","adjustedEnd":"0:01:25.419","start":"0:01:20.08","end":"0:01:25.419"}]},{"id":43,"name":"land vehicle","language":"en-US","instances":[{"confidence":0.9167,"adjustedStart":"0:01:21.415","adjustedEnd":"0:01:22.75","start":"0:01:21.415","end":"0:01:22.75"}]},{"id":44,"name":"walking","referenceId":"action/walk/walking","language":"en-US","instances":[{"confidence":0.8714,"adjustedStart":"0:01:21.415","adjustedEnd":"0:01:22.75","start":"0:01:21.415","end":"0:01:22.75"}]},{"id":45,"name":"business","referenceId":"enterprise/business","language":"en-US","instances":[{"confidence":0.9473,"adjustedStart":"0:01:22.749","adjustedEnd":"0:01:25.419","start":"0:01:22.749","end":"0:01:25.419"}]},{"id":46,"name":"wearing","language":"en-US","instances":[{"confidence":0.9131,"adjustedStart":"0:01:24.084","adjustedEnd":"0:01:25.419","start":"0:01:24.084","end":"0:01:25.419"}]},{"id":47,"name":"phone","referenceId":"telephone","language":"en-US","instances":[{"confidence":0.9535,"adjustedStart":"0:01:25.419","adjustedEnd":"0:01:26.754","start":"0:01:25.419","end":"0:01:26.754"}]},{"id":48,"name":"using","language":"en-US","instances":[{"confidence":0.9469,"adjustedStart":"0:01:25.419","adjustedEnd":"0:01:26.754","start":"0:01:25.419","end":"0:01:26.754"}]},{"id":49,"name":"talking","referenceId":"communication/talk/talking","language":"en-US","instances":[{"confidence":0.9393,"adjustedStart":"0:01:25.419","adjustedEnd":"0:01:26.754","start":"0:01:25.419","end":"0:01:26.754"}]},{"id":50,"name":"lady","referenceId":"person/woman/lady","language":"en-US","instances":[{"confidence":0.9304,"adjustedStart":"0:01:25.419","adjustedEnd":"0:01:28.088","start":"0:01:25.419","end":"0:01:28.088"}]},{"id":51,"name":"cellphone","referenceId":"electronic equipment/telephone/cellular telephone/cellphone","language":"en-US","instances":[{"confidence":0.912,"adjustedStart":"0:01:25.419","adjustedEnd":"0:01:26.754","start":"0:01:25.419","end":"0:01:26.754"}]},{"id":52,"name":"portrait","language":"en-US","instances":[{"confidence":0.9496,"adjustedStart":"0:01:25.419","adjustedEnd":"0:01:28.088","start":"0:01:25.419","end":"0:01:28.088"},{"confidence":0.986,"adjustedStart":"0:02:02.789","adjustedEnd":"0:02:04.124","start":"0:02:02.789","end":"0:02:04.124"}]},{"id":53,"name":"sitting","referenceId":"action/sitting","language":"en-US","instances":[{"confidence":0.8704,"adjustedStart":"0:01:28.088","adjustedEnd":"0:01:29.423","start":"0:01:28.088","end":"0:01:29.423"},{"confidence":0.8369,"adjustedStart":"0:01:30.757","adjustedEnd":"0:01:33.427","start":"0:01:30.757","end":"0:01:33.427"},{"confidence":0.8802,"adjustedStart":"0:01:36.096","adjustedEnd":"0:01:41.435","start":"0:01:36.096","end":"0:01:41.435"}]},{"id":54,"name":"fashion","language":"en-US","instances":[{"confidence":0.9851,"adjustedStart":"0:01:29.423","adjustedEnd":"0:01:30.758","start":"0:01:29.423","end":"0:01:30.758"}]},{"id":55,"name":"dress","referenceId":"consumer goods/clothing/dress","language":"en-US","instances":[{"confidence":0.9583,"adjustedStart":"0:01:29.423","adjustedEnd":"0:01:30.758","start":"0:01:29.423","end":"0:01:30.758"},{"confidence":0.9006,"adjustedStart":"0:02:06.793","adjustedEnd":"0:02:08.128","start":"0:02:06.793","end":"0:02:08.128"},{"confidence":0.9264,"adjustedStart":"0:02:09.463","adjustedEnd":"0:02:10.798","start":"0:02:09.463","end":"0:02:10.798"},{"confidence":0.9537,"adjustedStart":"0:02:12.132","adjustedEnd":"0:02:14.802","start":"0:02:12.132","end":"0:02:14.802"},{"confidence":0.9498,"adjustedStart":"0:02:17.471","adjustedEnd":"0:02:18.806","start":"0:02:17.471","end":"0:02:18.806"}]},{"id":56,"name":"footwear","referenceId":"consumer goods/clothing/footwear","language":"en-US","instances":[{"confidence":0.9126,"adjustedStart":"0:01:34.761","adjustedEnd":"0:01:36.096","start":"0:01:34.761","end":"0:01:36.096"}]},{"id":57,"name":"clothes hanger","language":"en-US","instances":[{"confidence":0.9382,"adjustedStart":"0:01:42.769","adjustedEnd":"0:01:44.104","start":"0:01:42.769","end":"0:01:44.104"}]},{"id":58,"name":"hanger","referenceId":"person/hanger","language":"en-US","instances":[{"confidence":0.876,"adjustedStart":"0:01:42.769","adjustedEnd":"0:01:44.104","start":"0:01:42.769","end":"0:01:44.104"}]},{"id":59,"name":"looking","referenceId":"look/looking","language":"en-US","instances":[{"confidence":0.8553,"adjustedStart":"0:01:49.443","adjustedEnd":"0:01:50.778","start":"0:01:49.443","end":"0:01:50.778"}]},{"id":60,"name":"dark","referenceId":"dark","language":"en-US","instances":[{"confidence":0.9894,"adjustedStart":"0:02:04.124","adjustedEnd":"0:02:05.459","start":"0:02:04.124","end":"0:02:05.459"}]},{"id":61,"name":"playing","referenceId":"performance/playing","language":"en-US","instances":[{"confidence":0.9496,"adjustedStart":"0:02:08.128","adjustedEnd":"0:02:09.463","start":"0:02:08.128","end":"0:02:09.463"}]},{"id":62,"name":"white","referenceId":"white","language":"en-US","instances":[{"confidence":0.9027,"adjustedStart":"0:02:14.801","adjustedEnd":"0:02:16.136","start":"0:02:14.801","end":"0:02:16.136"}]},{"id":63,"name":"smoke","referenceId":"smoke","language":"en-US","instances":[{"confidence":0.9669,"adjustedStart":"0:02:16.136","adjustedEnd":"0:02:17.471","start":"0:02:16.136","end":"0:02:17.471"}]},{"id":64,"name":"nature","referenceId":"nature","language":"en-US","instances":[{"confidence":0.9143,"adjustedStart":"0:02:16.136","adjustedEnd":"0:02:17.471","start":"0:02:16.136","end":"0:02:17.471"}]},{"id":65,"name":"window","referenceId":"structure/framework/window","language":"en-US","instances":[{"confidence":0.9437,"adjustedStart":"0:02:18.805","adjustedEnd":"0:02:21.475","start":"0:02:18.805","end":"0:02:21.475"}]},{"id":66,"name":"house","referenceId":"structure/building/house","language":"en-US","instances":[{"confidence":0.9446,"adjustedStart":"0:02:20.14","adjustedEnd":"0:02:21.475","start":"0:02:20.14","end":"0:02:21.475"}]},{"id":67,"name":"pillow","referenceId":"padding/cushion/pillow","language":"en-US","instances":[{"confidence":0.9186,"adjustedStart":"0:02:20.14","adjustedEnd":"0:02:21.475","start":"0:02:20.14","end":"0:02:21.475"}]},{"id":68,"name":"bedroom","referenceId":"structure/room/bedroom","language":"en-US","instances":[{"confidence":0.8463,"adjustedStart":"0:02:20.14","adjustedEnd":"0:02:21.475","start":"0:02:20.14","end":"0:02:21.475"}]},{"id":69,"name":"bed","referenceId":"furniture/bed","language":"en-US","instances":[{"confidence":0.8557,"adjustedStart":"0:02:20.14","adjustedEnd":"0:02:22.81","start":"0:02:20.14","end":"0:02:22.81"}]},{"id":70,"name":"screen","referenceId":"electronics/screen","language":"en-US","instances":[{"confidence":0.9697,"adjustedStart":"0:02:24.144","adjustedEnd":"0:02:25.479","start":"0:02:24.144","end":"0:02:25.479"},{"confidence":0.9689,"adjustedStart":"0:02:26.813","adjustedEnd":"0:02:28.148","start":"0:02:26.813","end":"0:02:28.148"},{"confidence":0.9714,"adjustedStart":"0:02:29.483","adjustedEnd":"0:02:32.152","start":"0:02:29.483","end":"0:02:32.152"}]},{"id":71,"name":"monitor","referenceId":"monitor","language":"en-US","instances":[{"confidence":0.9353,"adjustedStart":"0:02:24.144","adjustedEnd":"0:02:25.479","start":"0:02:24.144","end":"0:02:25.479"},{"confidence":0.9355,"adjustedStart":"0:02:26.813","adjustedEnd":"0:02:28.148","start":"0:02:26.813","end":"0:02:28.148"},{"confidence":0.9347,"adjustedStart":"0:02:29.483","adjustedEnd":"0:02:32.152","start":"0:02:29.483","end":"0:02:32.152"}]},{"id":72,"name":"remote","referenceId":"instrumentality/device/remote control/remote","language":"en-US","instances":[{"confidence":0.9332,"adjustedStart":"0:02:24.144","adjustedEnd":"0:02:25.479","start":"0:02:24.144","end":"0:02:25.479"}]},{"id":73,"name":"template","language":"en-US","instances":[{"confidence":0.986,"adjustedStart":"0:02:25.479","adjustedEnd":"0:02:29.483","start":"0:02:25.479","end":"0:02:29.483"}]},{"id":74,"name":"electronics","language":"en-US","instances":[{"confidence":0.9162,"adjustedStart":"0:02:26.813","adjustedEnd":"0:02:28.148","start":"0:02:26.813","end":"0:02:28.148"}]},{"id":75,"name":"internet","language":"en-US","instances":[{"confidence":0.9882,"adjustedStart":"0:02:30.817","adjustedEnd":"0:02:32.152","start":"0:02:30.817","end":"0:02:32.152"}]}],"scenes":[{"id":1,"instances":[{"adjustedStart":"0:00:00","adjustedEnd":"0:00:57.933","start":"0:00:00","end":"0:00:57.933"}]},{"id":2,"instances":[{"adjustedStart":"0:00:57.933","adjustedEnd":"0:01:19.454","start":"0:00:57.933","end":"0:01:19.454"}]},{"id":3,"instances":[{"adjustedStart":"0:01:19.454","adjustedEnd":"0:01:24.626","start":"0:01:19.454","end":"0:01:24.626"}]},{"id":4,"instances":[{"adjustedStart":"0:01:24.626","adjustedEnd":"0:02:06.376","start":"0:01:24.626","end":"0:02:06.376"}]},{"id":5,"instances":[{"adjustedStart":"0:02:06.376","adjustedEnd":"0:02:22.726","start":"0:02:06.376","end":"0:02:22.726"}]},{"id":6,"instances":[{"adjustedStart":"0:02:22.726","adjustedEnd":"0:02:24.227","start":"0:02:22.726","end":"0:02:24.227"}]},{"id":7,"instances":[{"adjustedStart":"0:02:24.227","adjustedEnd":"0:02:30.776","start":"0:02:24.227","end":"0:02:30.776"}]},{"id":8,"instances":[{"adjustedStart":"0:02:30.776","adjustedEnd":"0:02:32.444","start":"0:02:30.776","end":"0:02:32.444"}]}],"shots":[{"id":1,"tags":["Indoor","Medium","CenterFace"],"keyFrames":[{"id":1,"instances":[{"thumbnailId":"673ce1e9-5aa2-4df5-8ff2-e5cd77383d80","adjustedStart":"0:00:02.878","adjustedEnd":"0:00:02.92","start":"0:00:02.878","end":"0:00:02.92"}]},{"id":2,"instances":[{"thumbnailId":"aa643c02-827b-4c16-83d0-4b8f48b1a194","adjustedStart":"0:00:09.468","adjustedEnd":"0:00:09.51","start":"0:00:09.468","end":"0:00:09.51"}]},{"id":3,"instances":[{"thumbnailId":"d2410df7-0a77-4534-9469-3fba344d94fa","adjustedStart":"0:00:14.473","adjustedEnd":"0:00:14.515","start":"0:00:14.473","end":"0:00:14.515"}]}],"instances":[{"adjustedStart":"0:00:00","adjustedEnd":"0:00:19.144","start":"0:00:00","end":"0:00:19.144"}]},{"id":2,"tags":["Indoor"],"keyFrames":[{"id":4,"instances":[{"thumbnailId":"1c883233-86b1-4741-a648-533fbecd5742","adjustedStart":"0:00:19.144","adjustedEnd":"0:00:19.186","start":"0:00:19.144","end":"0:00:19.186"}]},{"id":5,"instances":[{"thumbnailId":"0c6c0fc3-75bb-4d24-a1aa-0faf1489a718","adjustedStart":"0:00:19.853","adjustedEnd":"0:00:19.895","start":"0:00:19.853","end":"0:00:19.895"}]},{"id":6,"instances":[{"thumbnailId":"6d39f23b-0417-4077-b2b5-065c7c56670e","adjustedStart":"0:00:21.605","adjustedEnd":"0:00:21.647","start":"0:00:21.605","end":"0:00:21.647"}]}],"instances":[{"adjustedStart":"0:00:19.144","adjustedEnd":"0:00:21.855","start":"0:00:19.144","end":"0:00:21.855"}]},{"id":3,"tags":["Indoor"],"keyFrames":[{"id":7,"instances":[{"thumbnailId":"7b2b3139-3c46-47f4-8776-9a82eb1b8298","adjustedStart":"0:00:21.855","adjustedEnd":"0:00:21.897","start":"0:00:21.855","end":"0:00:21.897"}]},{"id":8,"instances":[{"thumbnailId":"020ff10a-6ede-4d25-a0d5-7dbe672b1336","adjustedStart":"0:00:23.941","adjustedEnd":"0:00:23.983","start":"0:00:23.941","end":"0:00:23.983"}]}],"instances":[{"adjustedStart":"0:00:21.855","adjustedEnd":"0:00:27.194","start":"0:00:21.855","end":"0:00:27.194"}]},{"id":4,"tags":["Medium","RightFace","Indoor","Wide","CenterFace"],"keyFrames":[{"id":9,"instances":[{"thumbnailId":"1965ed6b-c89a-42ec-a715-a73ad46d53c7","adjustedStart":"0:00:27.194","adjustedEnd":"0:00:27.236","start":"0:00:27.194","end":"0:00:27.236"}]},{"id":10,"instances":[{"thumbnailId":"1da8e691-bb4a-436a-8464-796dfc82e48b","adjustedStart":"0:00:30.239","adjustedEnd":"0:00:30.281","start":"0:00:30.239","end":"0:00:30.281"}]}],"instances":[{"adjustedStart":"0:00:27.194","adjustedEnd":"0:00:34.743","start":"0:00:27.194","end":"0:00:34.743"}]},{"id":5,"tags":["Medium","RightFace","Indoor"],"keyFrames":[{"id":11,"instances":[{"thumbnailId":"863556b2-843c-4b3e-9444-8f4ef82f49b0","adjustedStart":"0:00:34.743","adjustedEnd":"0:00:34.785","start":"0:00:34.743","end":"0:00:34.785"}]}],"instances":[{"adjustedStart":"0:00:34.743","adjustedEnd":"0:00:36.411","start":"0:00:34.743","end":"0:00:36.411"}]},{"id":6,"tags":["Medium","LeftFace","Indoor"],"keyFrames":[{"id":12,"instances":[{"thumbnailId":"a866c913-34d3-4452-a3fd-c5340817e61d","adjustedStart":"0:00:36.411","adjustedEnd":"0:00:36.453","start":"0:00:36.411","end":"0:00:36.453"}]}],"instances":[{"adjustedStart":"0:00:36.411","adjustedEnd":"0:00:38.872","start":"0:00:36.411","end":"0:00:38.872"}]},{"id":7,"tags":["Medium","CenterFace","Indoor"],"keyFrames":[{"id":13,"instances":[{"thumbnailId":"f27c28f9-0ba7-4d02-a51f-7b9d76899682","adjustedStart":"0:00:38.872","adjustedEnd":"0:00:38.914","start":"0:00:38.872","end":"0:00:38.914"}]}],"instances":[{"adjustedStart":"0:00:38.872","adjustedEnd":"0:00:40.999","start":"0:00:38.872","end":"0:00:40.999"}]},{"id":8,"tags":["Medium","CenterFace","Indoor"],"keyFrames":[{"id":14,"instances":[{"thumbnailId":"d1be7174-0ad4-4dce-b4fd-23926fba5217","adjustedStart":"0:00:40.999","adjustedEnd":"0:00:41.041","start":"0:00:40.999","end":"0:00:41.041"}]}],"instances":[{"adjustedStart":"0:00:40.999","adjustedEnd":"0:00:43.377","start":"0:00:40.999","end":"0:00:43.377"}]},{"id":9,"tags":["Medium","RightFace","Indoor"],"keyFrames":[{"id":15,"instances":[{"thumbnailId":"e9582de3-6502-43cc-b713-9c0a0e2d325f","adjustedStart":"0:00:43.377","adjustedEnd":"0:00:43.419","start":"0:00:43.377","end":"0:00:43.419"}]}],"instances":[{"adjustedStart":"0:00:43.377","adjustedEnd":"0:00:45.337","start":"0:00:43.377","end":"0:00:45.337"}]},{"id":10,"tags":["Medium","Indoor"],"keyFrames":[{"id":16,"instances":[{"thumbnailId":"78dba84b-a8eb-48b6-ac31-fd8bf354ec65","adjustedStart":"0:00:45.337","adjustedEnd":"0:00:45.379","start":"0:00:45.337","end":"0:00:45.379"}]}],"instances":[{"adjustedStart":"0:00:45.337","adjustedEnd":"0:00:47.464","start":"0:00:45.337","end":"0:00:47.464"}]},{"id":11,"tags":["Medium","RightFace","Indoor"],"keyFrames":[{"id":17,"instances":[{"thumbnailId":"43e68dd8-902d-426f-ba22-9c8f35b1786b","adjustedStart":"0:00:47.464","adjustedEnd":"0:00:47.506","start":"0:00:47.464","end":"0:00:47.506"}]},{"id":18,"instances":[{"thumbnailId":"53d5f7dc-c791-4026-95a3-a62a00c37371","adjustedStart":"0:00:48.715","adjustedEnd":"0:00:48.757","start":"0:00:48.715","end":"0:00:48.757"}]}],"instances":[{"adjustedStart":"0:00:47.464","adjustedEnd":"0:00:49.299","start":"0:00:47.464","end":"0:00:49.299"}]},{"id":12,"tags":["Wide","LeftFace","Indoor"],"keyFrames":[{"id":19,"instances":[{"thumbnailId":"ae5d999e-4700-42c6-93a9-a27355a63e9e","adjustedStart":"0:00:49.299","adjustedEnd":"0:00:49.341","start":"0:00:49.299","end":"0:00:49.341"}]}],"instances":[{"adjustedStart":"0:00:49.299","adjustedEnd":"0:00:57.933","start":"0:00:49.299","end":"0:00:57.933"}]},{"id":13,"tags":["Indoor"],"keyFrames":[{"id":20,"instances":[{"thumbnailId":"ef0ce775-66e4-4260-b2c5-d83a03c86d1b","adjustedStart":"0:00:57.933","adjustedEnd":"0:00:57.975","start":"0:00:57.933","end":"0:00:57.975"}]},{"id":21,"instances":[{"thumbnailId":"3133bd60-b5f2-4b03-a455-7335673964b7","adjustedStart":"0:01:04.564","adjustedEnd":"0:01:04.606","start":"0:01:04.564","end":"0:01:04.606"}]},{"id":22,"instances":[{"thumbnailId":"cb102205-5623-4f33-b7cd-d6ac97e85330","adjustedStart":"0:01:09.736","adjustedEnd":"0:01:09.778","start":"0:01:09.736","end":"0:01:09.778"}]}],"instances":[{"adjustedStart":"0:00:57.933","adjustedEnd":"0:01:14.783","start":"0:00:57.933","end":"0:01:14.783"}]},{"id":14,"tags":["Indoor"],"keyFrames":[{"id":23,"instances":[{"thumbnailId":"c1b3f366-554d-4ac5-b135-2c44c06a4904","adjustedStart":"0:01:14.783","adjustedEnd":"0:01:14.825","start":"0:01:14.783","end":"0:01:14.825"}]},{"id":24,"instances":[{"thumbnailId":"a62fe2c3-56a2-4b82-bd1d-0dd333269796","adjustedStart":"0:01:16.034","adjustedEnd":"0:01:16.076","start":"0:01:16.034","end":"0:01:16.076"}]}],"instances":[{"adjustedStart":"0:01:14.783","adjustedEnd":"0:01:19.454","start":"0:01:14.783","end":"0:01:19.454"}]},{"id":15,"tags":["Indoor"],"keyFrames":[{"id":25,"instances":[{"thumbnailId":"dd97aa8f-c49f-42c8-94d0-1e563e9b2825","adjustedStart":"0:01:19.454","adjustedEnd":"0:01:19.496","start":"0:01:19.454","end":"0:01:19.496"}]}],"instances":[{"adjustedStart":"0:01:19.454","adjustedEnd":"0:01:24.626","start":"0:01:19.454","end":"0:01:24.626"}]},{"id":16,"tags":["CloseUp","CenterFace"],"keyFrames":[{"id":26,"instances":[{"thumbnailId":"b0c6ee4a-f41e-4ce7-aed3-51f98c0cc60f","adjustedStart":"0:01:24.626","adjustedEnd":"0:01:24.668","start":"0:01:24.626","end":"0:01:24.668"}]}],"instances":[{"adjustedStart":"0:01:24.626","adjustedEnd":"0:01:27.296","start":"0:01:24.626","end":"0:01:27.296"}]},{"id":17,"tags":["Medium","LeftFace"],"keyFrames":[{"id":27,"instances":[{"thumbnailId":"66b89cf9-96f3-4187-8493-ef0a306e7f0e","adjustedStart":"0:01:27.296","adjustedEnd":"0:01:27.338","start":"0:01:27.296","end":"0:01:27.338"}]}],"instances":[{"adjustedStart":"0:01:27.296","adjustedEnd":"0:01:41.226","start":"0:01:27.296","end":"0:01:41.226"}]},{"id":18,"tags":["Indoor","Wide","CenterFace"],"keyFrames":[{"id":28,"instances":[{"thumbnailId":"c77d762c-d3cf-49ed-aced-e8be398843fc","adjustedStart":"0:01:41.226","adjustedEnd":"0:01:41.268","start":"0:01:41.226","end":"0:01:41.268"}]},{"id":29,"instances":[{"thumbnailId":"74acba37-3172-43cd-a12a-f3c587dfe959","adjustedStart":"0:01:47.024","adjustedEnd":"0:01:47.066","start":"0:01:47.024","end":"0:01:47.066"}]},{"id":30,"instances":[{"thumbnailId":"ea88c59c-e8af-4980-8524-2128472496db","adjustedStart":"0:01:51.111","adjustedEnd":"0:01:51.153","start":"0:01:51.111","end":"0:01:51.153"}]}],"instances":[{"adjustedStart":"0:01:41.226","adjustedEnd":"0:01:54.615","start":"0:01:41.226","end":"0:01:54.615"}]},{"id":19,"tags":["Indoor","Medium","LeftFace"],"keyFrames":[{"id":31,"instances":[{"thumbnailId":"a7f9dcc8-30f2-4d12-9f1e-0b40736e305a","adjustedStart":"0:01:54.615","adjustedEnd":"0:01:54.657","start":"0:01:54.615","end":"0:01:54.657"}]},{"id":32,"instances":[{"thumbnailId":"9506556c-66cf-4873-ab8b-32b2b5cec658","adjustedStart":"0:02:03.415","adjustedEnd":"0:02:03.457","start":"0:02:03.415","end":"0:02:03.457"}]},{"id":33,"instances":[{"thumbnailId":"18b19629-ba71-4976-971c-e4c43553634d","adjustedStart":"0:02:05.292","adjustedEnd":"0:02:05.334","start":"0:02:05.292","end":"0:02:05.334"}]}],"instances":[{"adjustedStart":"0:01:54.615","adjustedEnd":"0:02:06.376","start":"0:01:54.615","end":"0:02:06.376"}]},{"id":20,"tags":["Indoor","Wide","LeftFace"],"keyFrames":[{"id":34,"instances":[{"thumbnailId":"94437385-ce14-4797-aeb3-5f7042d2b92a","adjustedStart":"0:02:06.376","adjustedEnd":"0:02:06.418","start":"0:02:06.376","end":"0:02:06.418"}]},{"id":35,"instances":[{"thumbnailId":"7b3f7f1c-1365-43a0-9f65-6450fdfdee3c","adjustedStart":"0:02:14.968","adjustedEnd":"0:02:15.01","start":"0:02:14.968","end":"0:02:15.01"}]},{"id":36,"instances":[{"thumbnailId":"8ee76227-8108-4c13-8074-225f82bba330","adjustedStart":"0:02:20.14","adjustedEnd":"0:02:20.182","start":"0:02:20.14","end":"0:02:20.182"}]}],"instances":[{"adjustedStart":"0:02:06.376","adjustedEnd":"0:02:22.726","start":"0:02:06.376","end":"0:02:22.726"}]},{"id":21,"tags":["Indoor"],"keyFrames":[{"id":37,"instances":[{"thumbnailId":"52a27126-90eb-4135-9146-9a016ca5ca34","adjustedStart":"0:02:22.726","adjustedEnd":"0:02:22.768","start":"0:02:22.726","end":"0:02:22.768"}]},{"id":38,"instances":[{"thumbnailId":"81d68d7d-c96d-4e9a-937a-f86631b31ec5","adjustedStart":"0:02:23.477","adjustedEnd":"0:02:23.519","start":"0:02:23.477","end":"0:02:23.519"}]}],"instances":[{"adjustedStart":"0:02:22.726","adjustedEnd":"0:02:24.227","start":"0:02:22.726","end":"0:02:24.227"}]},{"id":22,"keyFrames":[{"id":39,"instances":[{"thumbnailId":"3d0c5bc1-ddc5-4c76-8e47-e0638c92e52f","adjustedStart":"0:02:24.227","adjustedEnd":"0:02:24.269","start":"0:02:24.227","end":"0:02:24.269"}]},{"id":40,"instances":[{"thumbnailId":"4f72d596-dd24-4f49-bbc5-318c7f4b4ccf","adjustedStart":"0:02:25.02","adjustedEnd":"0:02:25.062","start":"0:02:25.02","end":"0:02:25.062"}]}],"instances":[{"adjustedStart":"0:02:24.227","adjustedEnd":"0:02:25.937","start":"0:02:24.227","end":"0:02:25.937"}]},{"id":23,"keyFrames":[{"id":41,"instances":[{"thumbnailId":"f66a0b64-fa4a-4a03-8254-3475446bfcfb","adjustedStart":"0:02:25.937","adjustedEnd":"0:02:25.979","start":"0:02:25.937","end":"0:02:25.979"}]},{"id":42,"instances":[{"thumbnailId":"8cca1b1e-642d-4d9b-9bf5-690bb0acbdc2","adjustedStart":"0:02:26.73","adjustedEnd":"0:02:26.772","start":"0:02:26.73","end":"0:02:26.772"}]}],"instances":[{"adjustedStart":"0:02:25.937","adjustedEnd":"0:02:28.69","start":"0:02:25.937","end":"0:02:28.69"}]},{"id":24,"keyFrames":[{"id":43,"instances":[{"thumbnailId":"5ff61583-60cd-4b5e-90de-2a233734fbcb","adjustedStart":"0:02:28.69","adjustedEnd":"0:02:28.732","start":"0:02:28.69","end":"0:02:28.732"}]},{"id":44,"instances":[{"thumbnailId":"3e30d1df-3bcc-4ce6-a81d-c9643f9f4873","adjustedStart":"0:02:29.524","adjustedEnd":"0:02:29.566","start":"0:02:29.524","end":"0:02:29.566"}]}],"instances":[{"adjustedStart":"0:02:28.69","adjustedEnd":"0:02:30.776","start":"0:02:28.69","end":"0:02:30.776"}]},{"id":25,"keyFrames":[{"id":45,"instances":[{"thumbnailId":"8098b3dd-5b21-4406-a469-15db717db889","adjustedStart":"0:02:30.776","adjustedEnd":"0:02:30.818","start":"0:02:30.776","end":"0:02:30.818"}]},{"id":46,"instances":[{"thumbnailId":"89bc33e0-85a8-4c33-b8d5-f177291d8522","adjustedStart":"0:02:31.735","adjustedEnd":"0:02:31.777","start":"0:02:31.735","end":"0:02:31.777"}]}],"instances":[{"adjustedStart":"0:02:30.776","adjustedEnd":"0:02:32.444","start":"0:02:30.776","end":"0:02:32.444"}]}],"sentiments":[{"id":1,"averageScore":0.5,"sentimentType":"Neutral","instances":[{"adjustedStart":"0:00:00","adjustedEnd":"0:01:41.43","start":"0:00:00","end":"0:01:41.43"},{"adjustedStart":"0:01:45.44","adjustedEnd":"0:02:09.67","start":"0:01:45.44","end":"0:02:09.67"},{"adjustedStart":"0:02:11.41","adjustedEnd":"0:02:32.509","start":"0:02:11.41","end":"0:02:32.509"}]},{"id":2,"averageScore":0.9529,"sentimentType":"Positive","instances":[{"adjustedStart":"0:01:41.43","adjustedEnd":"0:01:45.44","start":"0:01:41.43","end":"0:01:45.44"},{"adjustedStart":"0:02:09.67","adjustedEnd":"0:02:11.41","start":"0:02:09.67","end":"0:02:11.41"}]}],"emotions":[{"id":1,"type":"Joy","instances":[{"confidence":0.7011,"adjustedStart":"0:01:41.43","adjustedEnd":"0:01:45.44","start":"0:01:41.43","end":"0:01:45.44"},{"confidence":0.6513,"adjustedStart":"0:02:09.67","adjustedEnd":"0:02:11.41","start":"0:02:09.67","end":"0:02:11.41"}]}],"visualContentModeration":[{"id":1,"adultScore":0,"racyScore":0.8647,"instances":[{"adjustedStart":"0:00:23.941","adjustedEnd":"0:00:24.024","start":"0:00:23.941","end":"0:00:24.024"}]},{"id":2,"adultScore":0.0002,"racyScore":0.933,"instances":[{"adjustedStart":"0:02:05.292","adjustedEnd":"0:02:05.375","start":"0:02:05.292","end":"0:02:05.375"}]},{"id":3,"adultScore":0.0003,"racyScore":0.9264,"instances":[{"adjustedStart":"0:02:14.968","adjustedEnd":"0:02:20.14","start":"0:02:14.968","end":"0:02:20.14"}]}],"blocks":[{"id":0,"instances":[{"adjustedStart":"0:00:00","adjustedEnd":"0:00:33.79","start":"0:00:00","end":"0:00:33.79"}]},{"id":1,"instances":[{"adjustedStart":"0:00:33.79","adjustedEnd":"0:01:16.92","start":"0:00:33.79","end":"0:01:16.92"}]},{"id":2,"instances":[{"adjustedStart":"0:01:16.92","adjustedEnd":"0:01:48.42","start":"0:01:16.92","end":"0:01:48.42"}]},{"id":3,"instances":[{"adjustedStart":"0:01:48.42","adjustedEnd":"0:02:32.509","start":"0:01:48.42","end":"0:02:32.509"}]}],"framePatterns":[{"id":1,"patternType":"RollingCredits","confidence":0.97,"instances":[{"adjustedStart":"0:02:21","adjustedEnd":"0:02:32","start":"0:02:21","end":"0:02:32"}]},{"id":2,"patternType":"Black","confidence":1,"instances":[{"adjustedStart":"0:00:21.647","adjustedEnd":"0:00:21.73","start":"0:00:21.647","end":"0:00:21.73"}]}],"speakers":[{"id":1,"name":"Speaker #1","instances":[{"adjustedStart":"0:00:29.73","adjustedEnd":"0:00:46.64","start":"0:00:29.73","end":"0:00:46.64"}]},{"id":2,"name":"Speaker #2","instances":[{"adjustedStart":"0:00:49.57","adjustedEnd":"0:00:50.9","start":"0:00:49.57","end":"0:00:50.9"}]},{"id":3,"name":"Speaker #3","instances":[{"adjustedStart":"0:01:08.73","adjustedEnd":"0:01:16.92","start":"0:01:08.73","end":"0:01:16.92"},{"adjustedStart":"0:01:19.16","adjustedEnd":"0:01:24.84","start":"0:01:19.16","end":"0:01:24.84"},{"adjustedStart":"0:01:41.99","adjustedEnd":"0:01:53.02","start":"0:01:41.99","end":"0:01:53.02"}]},{"id":4,"name":"Speaker #4","instances":[{"adjustedStart":"0:01:24.84","adjustedEnd":"0:01:31.59","start":"0:01:24.84","end":"0:01:31.59"}]},{"id":5,"name":"Speaker #5","instances":[{"adjustedStart":"0:01:32.72","adjustedEnd":"0:01:41.99","start":"0:01:32.72","end":"0:01:41.99"}]},{"id":6,"name":"Speaker #6","instances":[{"adjustedStart":"0:01:53.02","adjustedEnd":"0:01:53.29","start":"0:01:53.02","end":"0:01:53.29"}]},{"id":7,"name":"Speaker #7","instances":[{"adjustedStart":"0:02:05.88","adjustedEnd":"0:02:11.41","start":"0:02:05.88","end":"0:02:11.41"}]},{"id":8,"name":"Speaker #8","instances":[{"adjustedStart":"0:02:11.41","adjustedEnd":"0:02:12.58","start":"0:02:11.41","end":"0:02:12.58"}]}],"textualContentModeration":{"id":0,"bannedWordsCount":0,"bannedWordsRatio":0,"instances":[]},"statistics":{"correspondenceCount":8,"speakerTalkToListenRatio":{"1":0.255,"2":0.02,"3":0.376,"4":0.102,"5":0.14,"6":0.004,"7":0.083,"8":0.017},"speakerLongestMonolog":{"1":16,"2":1,"3":13,"4":6,"5":9,"6":0,"7":5,"8":1},"speakerNumberOfFragments":{"1":1,"2":1,"3":3,"4":1,"5":1,"6":1,"7":1,"8":1},"speakerWordCount":{"1":57,"2":6,"3":69,"4":19,"5":33,"6":1,"7":19,"8":4}}},"thumbnailId":"344ad76c-d471-41db-a049-ba8233fe96d3","detectSourceLanguage":false,"languageAutoDetectMode":"None","sourceLanguage":"en-US","sourceLanguages":["en-US"],"language":"en-US","languages":["en-US"],"indexingPreset":"Default","linguisticModelId":"00000000-0000-0000-0000-000000000000","personModelId":"00000000-0000-0000-0000-000000000000","isAdult":false,"publishedUrl":"https://rodmandev.streaming.mediaservices.windows.net/7a8181b0-bc25-4934-9155-1e886ce8af6f/e13326ac-70c2-4fea-8aec-126f954d.ism/manifest(encryption=cbc)","publishedProxyUrl":null,"viewToken":"Bearer=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1cm46bWljcm9zb2Z0OmF6dXJlOm1lZGlhc2VydmljZXM6Y29udGVudGtleWlkZW50aWZpZXIiOiJjNWUzMTNlMS1lMWQ0LTQ0MjMtODkwOS0zN2I0MmViN2MxN2QiLCJuYmYiOjE1ODc1NDczMTYsImV4cCI6MTU4NzU5MDU3NiwiaXNzIjoiaHR0cHM6Ly9icmVha2Rvd24ubWUiLCJhdWQiOiJCcmVha2Rvd25Vc2VyIn0.KY0PndQ2EoLcgWbTQ5uPMgRqlpXHACet2Cx5ecLgDOA"}],"videosRanges":[{"videoId":"d9d45714f3","range":{"start":"0:00:00","end":"0:02:32.555"}}]} \ No newline at end of file diff --git a/controllers/potatoBall.json b/controllers/potatoBall.json new file mode 100644 index 0000000..30fd715 --- /dev/null +++ b/controllers/potatoBall.json @@ -0,0 +1 @@ +{"partition":null,"description":null,"privacyMode":"Public","state":"Processed","accountId":"5a8c6ddf-900f-48b4-8a3a-0cd5cf48cb4b","id":"7a9452bda8","name":"fried_potato_balls-nqXz8hhAYGo","userName":"Umang Mathur","created":"2019-07-26T18:38:21.3225566+00:00","isOwned":true,"isEditable":true,"isBase":true,"durationInSeconds":137,"summarizedInsights":{"name":"fried_potato_balls-nqXz8hhAYGo","id":"7a9452bda8","privacyMode":"Public","duration":{"time":"0:02:17.174","seconds":137.2},"thumbnailVideoId":"7a9452bda8","thumbnailId":"e0cc7550-60c1-4b5c-8a59-3ff3d8aff4c8","faces":[],"keywords":[{"isTranscript":false,"id":0,"name":"put potatoes","appearances":[{"startTime":"0:00:06.72","endTime":"0:00:08.64","startSeconds":6.7,"endSeconds":8.6},{"startTime":"0:00:26.4","endTime":"0:00:30.24","startSeconds":26.4,"endSeconds":30.2},{"startTime":"0:02:14.4","endTime":"0:02:14.56","startSeconds":134.4,"endSeconds":134.6}]},{"isTranscript":false,"id":1,"name":"paper towel","appearances":[{"startTime":"0:02:05.28","endTime":"0:02:06.24","startSeconds":125.3,"endSeconds":126.2}]},{"isTranscript":false,"id":2,"name":"ball shape","appearances":[{"startTime":"0:01:12.96","endTime":"0:01:14.88","startSeconds":73,"endSeconds":74.9},{"startTime":"0:02:14.4","endTime":"0:02:14.56","startSeconds":134.4,"endSeconds":134.6}]},{"isTranscript":false,"id":3,"name":"entire world","appearances":[{"startTime":"0:00:36.96","endTime":"0:00:38.4","startSeconds":37,"endSeconds":38.4}]},{"isTranscript":false,"id":4,"name":"finish shaping","appearances":[{"startTime":"0:01:43.68","endTime":"0:01:47.52","startSeconds":103.7,"endSeconds":107.5}]},{"isTranscript":false,"id":5,"name":"finish","appearances":[{"startTime":"0:01:43.68","endTime":"0:01:47.52","startSeconds":103.7,"endSeconds":107.5}]},{"isTranscript":false,"id":6,"name":"flour","appearances":[{"startTime":"0:01:21.12","endTime":"0:01:21.28","startSeconds":81.1,"endSeconds":81.3},{"startTime":"0:02:14.4","endTime":"0:02:14.56","startSeconds":134.4,"endSeconds":134.6}]},{"isTranscript":false,"id":7,"name":"bread","appearances":[{"startTime":"0:01:39.84","endTime":"0:01:42.24","startSeconds":99.8,"endSeconds":102.2},{"startTime":"0:02:14.4","endTime":"0:02:14.56","startSeconds":134.4,"endSeconds":134.6}]},{"isTranscript":false,"id":8,"name":"cover","appearances":[{"startTime":"0:01:04.8","endTime":"0:01:06.24","startSeconds":64.8,"endSeconds":66.2},{"startTime":"0:01:21.12","endTime":"0:01:21.28","startSeconds":81.1,"endSeconds":81.3}]},{"isTranscript":false,"id":9,"name":"cool","appearances":[{"startTime":"0:00:20.16","endTime":"0:00:24.96","startSeconds":20.2,"endSeconds":25},{"startTime":"0:01:04.8","endTime":"0:01:06.24","startSeconds":64.8,"endSeconds":66.2}]},{"isTranscript":false,"id":10,"name":"cooked","appearances":[{"startTime":"0:00:14.88","endTime":"0:00:18.24","startSeconds":14.9,"endSeconds":18.2}]},{"isTranscript":false,"id":11,"name":"plate","appearances":[{"startTime":"0:01:43.68","endTime":"0:01:45.6","startSeconds":103.7,"endSeconds":105.6}]},{"isTranscript":false,"id":12,"name":"check","appearances":[{"startTime":"0:00:14.88","endTime":"0:00:19.68","startSeconds":14.9,"endSeconds":19.7}]},{"isTranscript":false,"id":13,"name":"knife","appearances":[{"startTime":"0:00:14.88","endTime":"0:00:15.36","startSeconds":14.9,"endSeconds":15.4},{"startTime":"0:00:19.2","endTime":"0:00:19.68","startSeconds":19.2,"endSeconds":19.7}]},{"isTranscript":false,"id":14,"name":"salt","appearances":[{"startTime":"0:00:34.08","endTime":"0:00:34.56","startSeconds":34.1,"endSeconds":34.6},{"startTime":"0:01:27.36","endTime":"0:01:27.84","startSeconds":87.4,"endSeconds":87.8}]},{"isTranscript":false,"id":15,"name":"pinch","appearances":[{"startTime":"0:00:34.08","endTime":"0:00:34.56","startSeconds":34.1,"endSeconds":34.6},{"startTime":"0:01:29.76","endTime":"0:01:29.92","startSeconds":89.8,"endSeconds":89.9}]},{"isTranscript":false,"id":16,"name":"eating","appearances":[{"startTime":"0:02:05.28","endTime":"0:02:06.24","startSeconds":125.3,"endSeconds":126.2}]},{"isTranscript":false,"id":17,"name":"time","appearances":[{"startTime":"0:01:08.64","endTime":"0:01:11.04","startSeconds":68.6,"endSeconds":71}]},{"isTranscript":false,"id":18,"name":"teaspoon","appearances":[{"startTime":"0:01:08.64","endTime":"0:01:10.56","startSeconds":68.6,"endSeconds":70.6}]}],"sentiments":[],"emotions":[],"audioEffects":[],"labels":[{"id":1,"name":"food","appearances":[{"startTime":"0:00:01.28","endTime":"0:00:20.48","startSeconds":1.3,"endSeconds":20.5},{"startTime":"0:00:26.88","endTime":"0:01:59.04","startSeconds":26.9,"endSeconds":119},{"startTime":"0:02:02.88","endTime":"0:02:17.12","startSeconds":122.9,"endSeconds":137.1}]},{"id":11,"name":"plate","appearances":[{"startTime":"0:00:28.16","endTime":"0:00:37.12","startSeconds":28.2,"endSeconds":37.1},{"startTime":"0:00:42.24","endTime":"0:00:44.8","startSeconds":42.2,"endSeconds":44.8},{"startTime":"0:00:48.64","endTime":"0:01:05.28","startSeconds":48.6,"endSeconds":65.3},{"startTime":"0:01:11.68","endTime":"0:01:48.8","startSeconds":71.7,"endSeconds":108.8},{"startTime":"0:02:02.88","endTime":"0:02:09.28","startSeconds":122.9,"endSeconds":129.3},{"startTime":"0:02:16.96","endTime":"0:02:17.12","startSeconds":137,"endSeconds":137.1}]},{"id":3,"name":"indoor","appearances":[{"startTime":"0:00:06.4","endTime":"0:00:07.68","startSeconds":6.4,"endSeconds":7.7},{"startTime":"0:00:25.6","endTime":"0:00:26.88","startSeconds":25.6,"endSeconds":26.9},{"startTime":"0:00:34.56","endTime":"0:00:42.24","startSeconds":34.6,"endSeconds":42.2},{"startTime":"0:00:48.64","endTime":"0:00:57.6","startSeconds":48.6,"endSeconds":57.6},{"startTime":"0:01:09.12","endTime":"0:01:10.4","startSeconds":69.1,"endSeconds":70.4},{"startTime":"0:01:14.24","endTime":"0:01:18.08","startSeconds":74.2,"endSeconds":78.1},{"startTime":"0:01:36","endTime":"0:01:51.36","startSeconds":96,"endSeconds":111.4},{"startTime":"0:02:02.88","endTime":"0:02:09.28","startSeconds":122.9,"endSeconds":129.3}]},{"id":4,"name":"person","appearances":[{"startTime":"0:00:06.4","endTime":"0:00:07.68","startSeconds":6.4,"endSeconds":7.7},{"startTime":"0:00:37.12","endTime":"0:00:42.24","startSeconds":37.1,"endSeconds":42.2},{"startTime":"0:01:05.28","endTime":"0:01:23.2","startSeconds":65.3,"endSeconds":83.2},{"startTime":"0:01:30.88","endTime":"0:01:36","startSeconds":90.9,"endSeconds":96},{"startTime":"0:01:39.84","endTime":"0:01:44.96","startSeconds":99.8,"endSeconds":105}]},{"id":5,"name":"bowl","appearances":[{"startTime":"0:00:07.68","endTime":"0:00:11.52","startSeconds":7.7,"endSeconds":11.5},{"startTime":"0:00:15.36","endTime":"0:00:19.2","startSeconds":15.4,"endSeconds":19.2},{"startTime":"0:01:48.8","endTime":"0:01:55.2","startSeconds":108.8,"endSeconds":115.2}]},{"id":2,"name":"fruit","appearances":[{"startTime":"0:00:05.12","endTime":"0:00:06.4","startSeconds":5.1,"endSeconds":6.4},{"startTime":"0:00:28.16","endTime":"0:00:37.12","startSeconds":28.2,"endSeconds":37.1},{"startTime":"0:00:42.24","endTime":"0:00:43.52","startSeconds":42.2,"endSeconds":43.5}]},{"id":13,"name":"rice","appearances":[{"startTime":"0:00:56.32","endTime":"0:00:58.88","startSeconds":56.3,"endSeconds":58.9},{"startTime":"0:01:20.64","endTime":"0:01:25.76","startSeconds":80.6,"endSeconds":85.8}]},{"id":6,"name":"cup","appearances":[{"startTime":"0:00:11.52","endTime":"0:00:15.36","startSeconds":11.5,"endSeconds":15.4},{"startTime":"0:00:21.76","endTime":"0:00:24.32","startSeconds":21.8,"endSeconds":24.3},{"startTime":"0:01:06.56","endTime":"0:01:07.84","startSeconds":66.6,"endSeconds":67.8}]},{"id":15,"name":"table","appearances":[{"startTime":"0:01:20.64","endTime":"0:01:27.04","startSeconds":80.6,"endSeconds":87}]},{"id":0,"name":"dish","appearances":[{"startTime":"0:00:00","endTime":"0:00:05.12","startSeconds":0,"endSeconds":5.1}]},{"id":14,"name":"egg","appearances":[{"startTime":"0:00:58.88","endTime":"0:01:00.16","startSeconds":58.9,"endSeconds":60.2}]},{"id":7,"name":"kitchenware","appearances":[{"startTime":"0:00:14.08","endTime":"0:00:15.36","startSeconds":14.1,"endSeconds":15.4}]},{"id":8,"name":"soup","appearances":[{"startTime":"0:00:17.92","endTime":"0:00:19.2","startSeconds":17.9,"endSeconds":19.2}]},{"id":9,"name":"doughnut","appearances":[{"startTime":"0:00:24.32","endTime":"0:00:25.6","startSeconds":24.3,"endSeconds":25.6}]},{"id":10,"name":"donut","appearances":[{"startTime":"0:00:24.32","endTime":"0:00:25.6","startSeconds":24.3,"endSeconds":25.6}]},{"id":12,"name":"banana","appearances":[{"startTime":"0:00:42.24","endTime":"0:00:43.52","startSeconds":42.2,"endSeconds":43.5}]},{"id":16,"name":"porridge","appearances":[{"startTime":"0:01:57.76","endTime":"0:01:59.04","startSeconds":117.8,"endSeconds":119}]}],"framePatterns":[{"id":1,"name":"RollingCredits","appearances":[{"startTime":"0:02:09","endTime":"0:02:17","startSeconds":129,"endSeconds":137}]},{"id":2,"name":"Black","appearances":[{"startTime":"0:00:05.48","endTime":"0:00:05.64","startSeconds":5.5,"endSeconds":5.6},{"startTime":"0:02:09.36","endTime":"0:02:09.48","startSeconds":129.4,"endSeconds":129.5}]}],"brands":[],"namedLocations":[],"namedPeople":[],"statistics":{"correspondenceCount":0,"speakerTalkToListenRatio":{},"speakerLongestMonolog":{},"speakerNumberOfFragments":{},"speakerWordCount":{}},"topics":[{"referenceUrl":"https://en.wikipedia.org/wiki/Category:Food_additives","iptcName":null,"iabName":null,"confidence":0.6815,"id":1,"name":"Food additives","appearances":[{"startTime":"0:00:34.08","endTime":"0:00:34.56","startSeconds":34.1,"endSeconds":34.6},{"startTime":"0:01:27.36","endTime":"0:01:27.84","startSeconds":87.4,"endSeconds":87.8}]},{"referenceUrl":"https://en.wikipedia.org/wiki/Category:Lithics","iptcName":null,"iabName":null,"confidence":0.6778,"id":2,"name":"Lithics","appearances":[{"startTime":"0:00:14.88","endTime":"0:00:19.68","startSeconds":14.9,"endSeconds":19.7}]},{"referenceUrl":"https://en.wikipedia.org/wiki/Category:Wheat","iptcName":null,"iabName":null,"confidence":0.6561,"id":3,"name":"Wheat","appearances":[{"startTime":"0:01:21.12","endTime":"0:01:21.28","startSeconds":81.1,"endSeconds":81.3},{"startTime":"0:02:14.4","endTime":"0:02:14.56","startSeconds":134.4,"endSeconds":134.6}]},{"referenceUrl":"https://en.wikipedia.org/wiki/Category:Flour","iptcName":null,"iabName":null,"confidence":0.6561,"id":4,"name":"Flour","appearances":[{"startTime":"0:01:21.12","endTime":"0:01:21.28","startSeconds":81.1,"endSeconds":81.3},{"startTime":"0:02:14.4","endTime":"0:02:14.56","startSeconds":134.4,"endSeconds":134.6}]}]},"videos":[{"accountId":"5a8c6ddf-900f-48b4-8a3a-0cd5cf48cb4b","id":"7a9452bda8","state":"Processed","moderationState":"OK","reviewState":"None","privacyMode":"Public","processingProgress":"100%","failureCode":"None","failureMessage":"","externalId":null,"externalUrl":null,"metadata":null,"insights":{"version":"1.0.0.0","duration":"0:02:17.174","sourceLanguage":"en-US","sourceLanguages":["en-US"],"language":"en-US","languages":["en-US"],"ocr":[{"id":0,"text":"Ingredients for about 20 balls:","confidence":0.9275,"left":66,"top":87,"width":822,"height":59,"language":"en-US","instances":[{"adjustedStart":"0:02:14.4","adjustedEnd":"0:02:14.56","start":"0:02:14.4","end":"0:02:14.56"},{"adjustedStart":"0:02:14.88","adjustedEnd":"0:02:15.04","start":"0:02:14.88","end":"0:02:15.04"},{"adjustedStart":"0:02:15.36","adjustedEnd":"0:02:15.52","start":"0:02:15.36","end":"0:02:15.52"},{"adjustedStart":"0:02:15.84","adjustedEnd":"0:02:16","start":"0:02:15.84","end":"0:02:16"},{"adjustedStart":"0:02:16.32","adjustedEnd":"0:02:16.48","start":"0:02:16.32","end":"0:02:16.48"}]},{"id":1,"text":"- 2 medium potatoes","confidence":0.9576,"left":67,"top":223,"width":558,"height":59,"language":"en-US","instances":[{"adjustedStart":"0:02:14.4","adjustedEnd":"0:02:14.56","start":"0:02:14.4","end":"0:02:14.56"},{"adjustedStart":"0:02:14.88","adjustedEnd":"0:02:15.04","start":"0:02:14.88","end":"0:02:15.04"},{"adjustedStart":"0:02:15.36","adjustedEnd":"0:02:15.52","start":"0:02:15.36","end":"0:02:15.52"},{"adjustedStart":"0:02:15.84","adjustedEnd":"0:02:16","start":"0:02:15.84","end":"0:02:16"},{"adjustedStart":"0:02:16.32","adjustedEnd":"0:02:16.48","start":"0:02:16.32","end":"0:02:16.48"}]},{"id":2,"text":"- 2 egg yolks","confidence":0.937,"left":67,"top":359,"width":332,"height":59,"language":"en-US","instances":[{"adjustedStart":"0:02:14.4","adjustedEnd":"0:02:14.56","start":"0:02:14.4","end":"0:02:14.56"},{"adjustedStart":"0:02:14.88","adjustedEnd":"0:02:15.04","start":"0:02:14.88","end":"0:02:15.04"},{"adjustedStart":"0:02:15.36","adjustedEnd":"0:02:15.52","start":"0:02:15.36","end":"0:02:15.52"},{"adjustedStart":"0:02:15.84","adjustedEnd":"0:02:16","start":"0:02:15.84","end":"0:02:16"},{"adjustedStart":"0:02:16.32","adjustedEnd":"0:02:16.48","start":"0:02:16.32","end":"0:02:16.48"}]},{"id":3,"text":"- flour, egg and breadcrums, to bread them","confidence":0.9411,"left":66,"top":427,"width":1165,"height":59,"language":"en-US","instances":[{"adjustedStart":"0:02:14.4","adjustedEnd":"0:02:14.56","start":"0:02:14.4","end":"0:02:14.56"},{"adjustedStart":"0:02:14.88","adjustedEnd":"0:02:15.04","start":"0:02:14.88","end":"0:02:15.04"},{"adjustedStart":"0:02:15.36","adjustedEnd":"0:02:15.52","start":"0:02:15.36","end":"0:02:15.52"},{"adjustedStart":"0:02:15.84","adjustedEnd":"0:02:16","start":"0:02:15.84","end":"0:02:16"},{"adjustedStart":"0:02:16.32","adjustedEnd":"0:02:16.48","start":"0:02:16.32","end":"0:02:16.48"}]},{"id":4,"text":"olive oil","confidence":0.9862,"left":104,"top":496,"width":211,"height":46,"language":"en-US","instances":[{"adjustedStart":"0:02:14.4","adjustedEnd":"0:02:14.56","start":"0:02:14.4","end":"0:02:14.56"},{"adjustedStart":"0:02:14.88","adjustedEnd":"0:02:15.04","start":"0:02:14.88","end":"0:02:15.04"},{"adjustedStart":"0:02:15.36","adjustedEnd":"0:02:15.52","start":"0:02:15.36","end":"0:02:15.52"},{"adjustedStart":"0:02:15.84","adjustedEnd":"0:02:16","start":"0:02:15.84","end":"0:02:16"},{"adjustedStart":"0:02:16.32","adjustedEnd":"0:02:16.48","start":"0:02:16.32","end":"0:02:16.48"}]},{"id":5,"text":"Now cover and let cool","confidence":0.8564,"left":322,"top":554,"width":572,"height":103,"language":"en-US","instances":[{"adjustedStart":"0:01:04.8","adjustedEnd":"0:01:06.24","start":"0:01:04.8","end":"0:01:06.24"},{"adjustedStart":"0:01:06.72","adjustedEnd":"0:01:08.16","start":"0:01:06.72","end":"0:01:08.16"}]},{"id":6,"text":"after abouvo 3q minutes, then tney are cooked","confidence":0.8437,"left":48,"top":564,"width":1161,"height":62,"language":"en-US","instances":[{"adjustedStart":"0:00:15.36","adjustedEnd":"0:00:18.24","start":"0:00:15.36","end":"0:00:18.24"},{"adjustedStart":"0:00:18.72","adjustedEnd":"0:00:19.2","start":"0:00:18.72","end":"0:00:19.2"}]},{"id":7,"text":"after abouvo minutes, they are cooked","confidence":0.9007,"left":48,"top":569,"width":1161,"height":57,"language":"en-US","instances":[{"adjustedStart":"0:00:14.88","adjustedEnd":"0:00:15.36","start":"0:00:14.88","end":"0:00:15.36"}]},{"id":8,"text":"...and black pepper.","confidence":0.8781,"left":382,"top":580,"width":492,"height":55,"language":"en-US","instances":[{"adjustedStart":"0:00:36.96","adjustedEnd":"0:00:41.28","start":"0:00:36.96","end":"0:00:41.28"}]},{"id":9,"text":"00","confidence":0.6445,"left":879,"top":595,"width":30,"height":19,"language":"en-US","instances":[{"adjustedStart":"0:00:35.04","adjustedEnd":"0:00:35.52","start":"0:00:35.04","end":"0:00:35.52"},{"adjustedStart":"0:00:36","adjustedEnd":"0:00:36.48","start":"0:00:36","end":"0:00:36.48"}]},{"id":10,"text":"Finish shåping and leåiké On)yplate","confidence":0.6372,"left":188,"top":608,"width":889,"height":65,"language":"en-US","instances":[{"adjustedStart":"0:01:43.68","adjustedEnd":"0:01:44.16","start":"0:01:43.68","end":"0:01:44.16"}]},{"id":11,"text":"Finish shåpipg and-Iqa.v.é\\.•önl@plate","confidence":0.6897,"left":188,"top":608,"width":889,"height":66,"language":"en-US","instances":[{"adjustedStart":"0:01:44.64","adjustedEnd":"0:01:45.12","start":"0:01:44.64","end":"0:01:45.12"}]},{"id":12,"text":"on by oil...","confidence":0.6743,"left":265,"top":609,"width":838,"height":57,"language":"en-US","instances":[{"adjustedStart":"0:01:54.24","adjustedEnd":"0:01:54.72","start":"0:01:54.24","end":"0:01:54.72"}]},{"id":13,"text":"on oil...","confidence":0.7,"left":265,"top":609,"width":837,"height":49,"language":"en-US","instances":[{"adjustedStart":"0:01:55.2","adjustedEnd":"0:01:55.68","start":"0:01:55.2","end":"0:01:55.68"}]},{"id":14,"text":"Finish shaping and a","confidence":0.999,"left":187,"top":611,"width":751,"height":58,"language":"en-US","instances":[{"adjustedStart":"0:01:46.56","adjustedEnd":"0:01:47.52","start":"0:01:46.56","end":"0:01:47.52"}]},{"id":15,"text":"Finish ,sha+ing and plate,","confidence":0.742,"left":188,"top":611,"width":901,"height":57,"language":"en-US","instances":[{"adjustedStart":"0:01:44.16","adjustedEnd":"0:01:44.64","start":"0:01:44.16","end":"0:01:44.64"}]},{"id":16,"text":"Finish shaping and leaveøon a","confidence":0.9444,"left":186,"top":612,"width":752,"height":59,"language":"en-US","instances":[{"adjustedStart":"0:01:46.08","adjustedEnd":"0:01:46.56","start":"0:01:46.08","end":"0:01:46.56"},{"adjustedStart":"0:01:47.52","adjustedEnd":"0:01:48","start":"0:01:47.52","end":"0:01:48"}]},{"id":17,"text":"ool","confidence":0.543,"left":838,"top":613,"width":78,"height":48,"language":"en-US","instances":[{"adjustedStart":"0:00:19.68","adjustedEnd":"0:00:20.16","start":"0:00:19.68","end":"0:00:20.16"}]},{"id":18,"text":"Finish shapmg a plate","confidence":0.8575,"left":188,"top":614,"width":889,"height":55,"language":"en-US","instances":[{"adjustedStart":"0:01:45.12","adjustedEnd":"0:01:45.6","start":"0:01:45.12","end":"0:01:45.6"}]},{"id":19,"text":"drain them anVd-let cool","confidence":0.6599,"left":327,"top":615,"width":587,"height":49,"language":"en-US","instances":[{"adjustedStart":"0:00:20.16","adjustedEnd":"0:00:24.48","start":"0:00:20.16","end":"0:00:24.48"}]},{"id":20,"text":"cool","confidence":0.989,"left":812,"top":616,"width":101,"height":43,"language":"en-US","instances":[{"adjustedStart":"0:00:24.48","adjustedEnd":"0:00:24.96","start":"0:00:24.48","end":"0:00:24.96"}]},{"id":21,"text":"—Béaten egg, with a„pinch of*alt","confidence":0.6492,"left":170,"top":618,"width":835,"height":71,"language":"en-US","instances":[{"adjustedStart":"0:01:29.76","adjustedEnd":"0:01:29.92","start":"0:01:29.76","end":"0:01:29.92"}]},{"id":22,"text":"(check!t a knife)...","confidence":0.7107,"left":331,"top":619,"width":590,"height":101,"language":"en-US","instances":[{"adjustedStart":"0:00:19.2","adjustedEnd":"0:00:19.68","start":"0:00:19.2","end":"0:00:19.68"}]},{"id":23,"text":"and cover with flour","confidence":0.8609,"left":409,"top":619,"width":510,"height":46,"language":"en-US","instances":[{"adjustedStart":"0:01:21.12","adjustedEnd":"0:01:21.28","start":"0:01:21.12","end":"0:01:21.28"},{"adjustedStart":"0:01:22.56","adjustedEnd":"0:01:22.72","start":"0:01:22.56","end":"0:01:22.72"}]},{"id":24,"text":"Bé*aJen egg, with a•ßinch of salt; r","confidence":0.6244,"left":218,"top":620,"width":835,"height":64,"language":"en-US","instances":[{"adjustedStart":"0:01:27.36","adjustedEnd":"0:01:27.84","start":"0:01:27.36","end":"0:01:27.84"}]},{"id":25,"text":"And breadyümbs;q","confidence":0.6995,"left":413,"top":620,"width":500,"height":66,"language":"en-US","instances":[{"adjustedStart":"0:01:39.84","adjustedEnd":"0:01:42.24","start":"0:01:39.84","end":"0:01:42.24"}]},{"id":26,"text":"but not too wrthiill-burst","confidence":0.6567,"left":189,"top":622,"width":890,"height":50,"language":"en-US","instances":[{"adjustedStart":"0:01:56.16","adjustedEnd":"0:01:56.8","start":"0:01:56.16","end":"0:01:56.8"},{"adjustedStart":"0:01:57.6","adjustedEnd":"0:01:57.76","start":"0:01:57.6","end":"0:01:57.76"}]},{"id":27,"text":"put potatoes in watuænrd•uoil them","confidence":0.8287,"left":174,"top":624,"width":904,"height":60,"language":"en-US","instances":[{"adjustedStart":"0:00:06.72","adjustedEnd":"0:00:07.2","start":"0:00:06.72","end":"0:00:07.2"},{"adjustedStart":"0:00:08.64","adjustedEnd":"0:00:09.6","start":"0:00:08.64","end":"0:00:09.6"}]},{"id":28,"text":"put potatoes in them","confidence":0.947,"left":174,"top":624,"width":904,"height":57,"language":"en-US","instances":[{"adjustedStart":"0:00:07.2","adjustedEnd":"0:00:08.64","start":"0:00:07.2","end":"0:00:08.64"},{"adjustedStart":"0:00:09.6","adjustedEnd":"0:00:12.48","start":"0:00:09.6","end":"0:00:12.48"}]},{"id":29,"text":"Andlbrehdcrumbss","confidence":0.436,"left":413,"top":625,"width":465,"height":55,"language":"en-US","instances":[{"adjustedStart":"0:01:37.44","adjustedEnd":"0:01:39.84","start":"0:01:37.44","end":"0:01:39.84"}]},{"id":30,"text":"...a pinch of salt...","confidence":0.9777,"left":383,"top":627,"width":441,"height":56,"language":"en-US","instances":[{"adjustedStart":"0:00:34.08","adjustedEnd":"0:00:34.56","start":"0:00:34.08","end":"0:00:34.56"},{"adjustedStart":"0:00:35.04","adjustedEnd":"0:00:35.52","start":"0:00:35.04","end":"0:00:35.52"},{"adjustedStart":"0:01:31.68","adjustedEnd":"0:01:31.84","start":"0:01:31.68","end":"0:01:31.84"}]},{"id":31,"text":"'zåndsleave on paper towel before eating","confidence":0.7939,"left":132,"top":628,"width":1007,"height":55,"language":"en-US","instances":[{"adjustedStart":"0:02:05.28","adjustedEnd":"0:02:05.76","start":"0:02:05.28","end":"0:02:05.76"},{"adjustedStart":"0:02:06.24","adjustedEnd":"0:02:06.72","start":"0:02:06.24","end":"0:02:06.72"},{"adjustedStart":"0:02:08.16","adjustedEnd":"0:02:08.64","start":"0:02:08.16","end":"0:02:08.64"}]},{"id":32,"text":"•When'byowned, remove from the oil","confidence":0.8806,"left":161,"top":628,"width":941,"height":65,"language":"en-US","instances":[{"adjustedStart":"0:02:01.92","adjustedEnd":"0:02:02.08","start":"0:02:01.92","end":"0:02:02.08"},{"adjustedStart":"0:02:02.4","adjustedEnd":"0:02:02.56","start":"0:02:02.4","end":"0:02:02.56"}]},{"id":33,"text":"on paper towel before eating","confidence":0.9552,"left":418,"top":628,"width":721,"height":55,"language":"en-US","instances":[{"adjustedStart":"0:02:05.76","adjustedEnd":"0:02:06.24","start":"0:02:05.76","end":"0:02:06.24"},{"adjustedStart":"0:02:06.72","adjustedEnd":"0:02:08.16","start":"0:02:06.72","end":"0:02:08.16"}]},{"id":34,"text":"If you enjoied","confidence":0.8772,"left":287,"top":630,"width":347,"height":55,"language":"en-US","instances":[{"adjustedStart":"0:02:11.04","adjustedEnd":"0:02:12","start":"0:02:11.04","end":"0:02:12"}]},{"id":35,"text":"give shape","confidence":0.7425,"left":418,"top":631,"width":371,"height":58,"language":"en-US","instances":[{"adjustedStart":"0:01:12.96","adjustedEnd":"0:01:13.44","start":"0:01:12.96","end":"0:01:13.44"}]},{"id":36,"text":"give ball shape","confidence":0.804,"left":418,"top":631,"width":371,"height":58,"language":"en-US","instances":[{"adjustedStart":"0:01:14.4","adjustedEnd":"0:01:14.88","start":"0:01:14.4","end":"0:01:14.88"},{"adjustedStart":"0:01:15.84","adjustedEnd":"0:01:16.8","start":"0:01:15.84","end":"0:01:16.8"},{"adjustedStart":"0:01:17.28","adjustedEnd":"0:01:17.44","start":"0:01:17.28","end":"0:01:17.44"},{"adjustedStart":"0:01:17.76","adjustedEnd":"0:01:17.92","start":"0:01:17.76","end":"0:01:17.92"},{"adjustedStart":"0:01:18.24","adjustedEnd":"0:01:18.4","start":"0:01:18.24","end":"0:01:18.4"}]},{"id":37,"text":"then put them into a bowl peeled and chopped","confidence":0.9867,"left":44,"top":633,"width":1179,"height":56,"language":"en-US","instances":[{"adjustedStart":"0:00:26.4","adjustedEnd":"0:00:30.24","start":"0:00:26.4","end":"0:00:30.24"}]},{"id":38,"text":"After this time, take a teaspqpu offååu(gliz-.","confidence":0.8144,"left":107,"top":633,"width":1090,"height":80,"language":"en-US","instances":[{"adjustedStart":"0:01:10.56","adjustedEnd":"0:01:11.04","start":"0:01:10.56","end":"0:01:11.04"}]},{"id":39,"text":"(Checktitawi!tln a-knife)'.","confidence":0.379,"left":330,"top":634,"width":603,"height":53,"language":"en-US","instances":[{"adjustedStart":"0:00:14.88","adjustedEnd":"0:00:15.36","start":"0:00:14.88","end":"0:00:15.36"}]},{"id":40,"text":"(Cheek it withea","confidence":0.6673,"left":330,"top":634,"width":383,"height":53,"language":"en-US","instances":[{"adjustedStart":"0:00:15.36","adjustedEnd":"0:00:15.84","start":"0:00:15.36","end":"0:00:15.84"},{"adjustedStart":"0:00:18.72","adjustedEnd":"0:00:19.2","start":"0:00:18.72","end":"0:00:19.2"}]},{"id":41,"text":"(Check itåwith aæk-n.iIfé)Z.","confidence":0.5582,"left":330,"top":634,"width":604,"height":53,"language":"en-US","instances":[{"adjustedStart":"0:00:16.32","adjustedEnd":"0:00:16.8","start":"0:00:16.32","end":"0:00:16.8"},{"adjustedStart":"0:00:17.28","adjustedEnd":"0:00:17.76","start":"0:00:17.28","end":"0:00:17.76"}]},{"id":42,"text":"@heckit with a","confidence":0.645,"left":330,"top":634,"width":383,"height":53,"language":"en-US","instances":[{"adjustedStart":"0:00:17.76","adjustedEnd":"0:00:18.24","start":"0:00:17.76","end":"0:00:18.24"}]},{"id":43,"text":"(Check{ + console.log(result); + }) + .catch(e => { + console.log(e); + }) + } + + + static async fetchAudioInsight(audioId, videoId){ + return new Promise((resolve, reject) => { + let url = 'http://18.221.192.73:8081/v1/audioclips/getspeechtotextresult?orderid=' + audioId; + axios.get(url) + .then((response) => { + console.log(response.data); + let status = response.data.status + if(status !== 'success'){ + reject(`Audio not processed for audioId=${audioId}, videoId= ${videoId}`); + } + //VIDEO + let videoInsight = []; + scene.getSceneData(videoId) + .then((data) => { + console.log(data); + for (let i = 0; i < data.length; i++) { + data[i].merged = []; + videoInsight.push(data[i]); + } + console.log({ videoInsight }); + + //AUDIO + let sentence = response.data.result.sentence; + let audioInsight =[]; + for(let i=0; i< sentence.length; i++){ + let start = sentence[i].startTime; + let end = sentence[i].endTime; + let desc = sentence[i].sentence; + desc = desc.replace(/'/g, ''); + if(!desc.toLowerCase().includes('noise')){ + let data = { + start, + end, + sentence: desc + } + audioInsight.push(data); + } + } + console.log({audioInsight}); + + //MERGE + let mergedList = this.merge(videoInsight, audioInsight); + scene.addMergeSceneTable(mergedList) + .then(data => { + console.log({ success: data }); + videoDialogue.insertDataIntoVideoDialogue(audioInsight, videoId, audioId) + .then(res=>{ + console.log(res); + resolve(); + }) + }).catch((error) => { + console.log(error); + reject(error); + }) + + }) + + }) + .catch((error) => { + console.log(error); + reject(error); + }) + }); + + } + + + + static merge(videoInsight, audioInsight){ + let videoIndex = 0; + let mergedList = []; + mergedList.push(videoInsight[videoIndex++]); + let i = 0; + while(i < audioInsight.length && videoIndex < videoInsight.length){ + let size = mergedList.length; + let s1 = mergedList[size-1].start_time; + let e1 = mergedList[size-1].end_time; + let description = mergedList[size-1].original_description; + let ocr = mergedList[size-1].original_ocr; + + let s2 = audioInsight[i].start; + let e2 = audioInsight[i].end; + + if((s2 >= s1 && s2 <= e1) && (e2 >= s1 && e2 <= e1)) { + i++; + continue; + } + + if(Math.min(s2, e2) >= e1) { + mergedList.push(videoInsight[videoIndex++]); + continue; + } + + //merge condition + if(s2 >= s1 && e2 > e1) { + let sceneObj = mergedList[mergedList.length-1]; + let next = videoInsight[videoIndex++]; + let min = Math.min(s1, next.start_time); + let max = Math.max(e1, next.end_time); + sceneObj.start_time = min; + sceneObj.end_time = max; + sceneObj.original_description = description + next.original_description; + sceneObj.original_ocr = ocr + next.original_ocr; + + // let arr = sceneObj.merged; + // arr.push(next.scene_id); + sceneObj.merged.push(next.scene_id); + mergedList.splice(mergedList.length-1); + mergedList.push(sceneObj); + }else if(s2 >= e1) { + mergedList.push(videoInsight[videoIndex++]); + } + + i++; + } + + for(let j = videoIndex; j', end); + } + console.log('video:', arr); + + return arr; + } + + static parseAudioFile(file){ + console.log("here"); + let rawdata = fs.readFileSync(file); + let obj = JSON.parse(rawdata); + let sentence = obj.result.sentence; + let arr =[]; + for(let i=0; i< sentence.length; i++){ + // let instance = scenes[i].instances[0]; + let start = sentence[i].startTime; + let end = sentence[i].endTime; + // let end = Utils.parseSecondsFromString(instance.end); + // console.log(start,'->', end); + // console.log(start , ',' , end); + let data = { + start, + end + } + arr.push(data); + } + + console.log(arr); + return arr; + } + + + static convertVideoToAudio(path, fileName){ + let audioFile = path + '/' + fileName+'.mp3'; + let videoFile = path + '/' + fileName + '.mp4' + console.log({path}); + console.log({audioFile}); + console.log({videoFile}); + + try { + var process = new ffmpeg(videoFile); + process.then(function (video) { + console.log({video}); + video.fnExtractSoundToMP3(audioFile, function (error, file) { + if (!error) + console.log('Audio file: ' + file); + }); + }, function (err) { + console.log('Error: ' + err); + }); + } catch (e) { + console.log('err->', e); + } + } + + + static async insertIntoVideoDialogue1(){ + let audioId = 'OMHPXC20200411053439'; + let videoId = 'vBkBS4O3yvY'; + + try{ + + let url = 'http://18.221.192.73:8081/v1/audioclips/getspeechtotextresult?orderid=' + audioId; + let response = await axios.get(url); + + //AUDIO + let sentence = response.data.result.sentence; + let audioInsight =[]; + for(let i=0; i< sentence.length; i++){ + let start = sentence[i].startTime; + let end = sentence[i].endTime; + let desc = sentence[i].sentence; + desc = desc.replace(/'/g, ''); + + let data = { + start, + end, + sentence: desc + } + audioInsight.push(data); + } + console.log({audioInsight}); + + await videoDialogue.insertDataIntoVideoDialogue(audioInsight, videoId, audioId) + + console.log('inserted'); + }catch (e){ + console.log(e); + } + + + + + } + + + async copyFromMergeToDescription(user, videoId){ + // let videoId = 'vBkBS4O3yvY'; + // let user = 'd32f61aa-7f67-11ea-bc55-0242ac130003'; + + console.log({videoId}); + try { + let data = await db.query('select * from merge_scene where video_id = $1 order by start_time', [videoId]); + // console.log(data); + console.log('here'); + let sql = ''; + for(let i=0; i { +// file = res.value; +// } +// ) +// form.append("file",file); +// axios.post('splitMp3', form, { +// headers: { +// 'Content-Type': 'multipart/form-data' +// } +// }) + + +const db = require('./db/database'); + +async function generateTimeGaps(vid_id, vol_id){ +try{ + + await db.any(`Delete from video_time_gaps where video_id = '${vid_id}'`); + +} catch (error) { + console.log(error); + return false; +} + +let descriptionListArr = await db.any(`SELECT * FROM descriptions WHERE video_id = '${vid_id}' AND volunteer_id = '${vol_id}'order by scene_num`); + +let scenes = {}; + +for(let i=0; i timestamp) + return sceneNumArr[i-1]['scene_num']; + } + + return sceneNumArr[sceneNumArr.length - 1]['scene_num']; +} + + +async function generateTimeGapsOnlyMusic(vid_id, vol_id){ + try{ + + await db.any(`Delete from video_time_gaps where video_id = '${vid_id}'`); + + } catch (error) { + console.log(error); + return false; + } + + let descriptionListArr = await db.any(`SELECT * FROM descriptions WHERE video_id = '${vid_id}' AND volunteer_id = '${vol_id}'order by scene_num`); + + let currSceneNum = []; + let curr_startTime = []; + let curr_endTime = []; + + for(let i=0; i { + sql += `UPDATE keyframe SET pythia_caption = '${data.caption}' + WHERE keyframe_id = '${data.keyframe_id}';`; + + }); + sql = 'START TRANSACTION;' + sql + 'COMMIT;' + console.log({sql}); + + return await db.query(sql); + } catch(e) { + throw e; + } + } + + + static async getKeyFramesByVideoId(videoId) { + console.log('get KeyFrames by video Id:', videoId); + try { + let sql = `select k.keyframe_id , s2.scene_id, k.shot_id, k.keyframe_num, k.start_time, k.end_time, k.thumbnail_id, + k.thumbnail_url, k.caption,k.caption_confidence, k.pythia_caption + from keyframe k , shot s, scene s2 + where k.shot_id = s.shot_id and s.scene_id = s2.scene_id + and s2.video_id = $1 order by k.start_time`; + + return await db.query(sql, [videoId]); + } catch(e) { + console.log({'error': e}); + throw e; + } + } + } module.exports = Keyframe; \ No newline at end of file diff --git a/model/keyframeRevise.js b/model/keyframeRevise.js new file mode 100644 index 0000000..c660233 --- /dev/null +++ b/model/keyframeRevise.js @@ -0,0 +1,25 @@ +const db = require('../db/database'); + +let keyframeRevise = { + + addKeyframe: function(data, callback) { + + const sql = `START TRANSACTION; + INSERT into keyframe_revised(keyframe_id, video_id, keyframe_num, keyframe_url, timestamp, pythia_caption) VALUES('${data.keyframeId}','${data.videoId}',${data.keyframeNum}, '${data.keyframeURL}', ${data.timestamp}, '${data.caption}'); + COMMIT;`; + console.log({sql}); + return db.query(sql, callback); + }, + + updateKeyframe: function(data, callback) { + + const sql = `START TRANSACTION; + INSERT into keyframe_revised (keyframe_id, video_id, keyframe_num, keyframe_url, timestamp) VALUES ('${data.keyframeId}','${data.videoId}',${data.keyframeNum}, '${data.keyframeURL}', ${data.timestamp}); + COMMIT;`; + + return db.query(sql, callback); + } +} + + +module.exports = keyframeRevise; \ No newline at end of file diff --git a/model/ocr.js b/model/ocr.js index aae2051..c290228 100644 --- a/model/ocr.js +++ b/model/ocr.js @@ -53,6 +53,20 @@ class Ocr { throw e; } } + + static async updateShotTableWithFilteredOcr(concatenatedOcrTextArr) { + console.log('DB Operation: Updating Shot table with Filtered OCR'); + try { + return await db.tx(t => { + var queries = concatenatedOcrTextArr.map(ocrObj => { + return t.one('UPDATE shot SET filtered_ocr = ${text} WHERE shot_id = ${shot_id} RETURNING id', ocrObj, a => + a.id); + }); + return t.batch(queries); + }); + } catch(e) { + throw e; + } + } } module.exports = Ocr; \ No newline at end of file diff --git a/model/questionAnswer.js b/model/questionAnswer.js index 27ed11e..6d69e2d 100644 --- a/model/questionAnswer.js +++ b/model/questionAnswer.js @@ -63,9 +63,9 @@ let question = { deleteQuestionById : function(data, callback){ // console.log('inside delete'); const sql = `START TRANSACTION; + Delete from scene_qna_mapping where question_id = '${data.questionId}' AND answer_id = '${data.answerId}' ; Delete from question where id = '${data.questionId}'; Delete from answer where id = '${data.answerId}'; - Delete from scene_qna_mapping where question_id = '${data.questionId}' AND answer_id = '${data.answerId}' ; COMMIT;`; // console.log({sql}); return db.query(sql, callback); diff --git a/model/questionUserStudy4.js b/model/questionUserStudy4.js new file mode 100644 index 0000000..b6c0fba --- /dev/null +++ b/model/questionUserStudy4.js @@ -0,0 +1,99 @@ +const db = require('../db/database'); +const uuidv1 = require('uuid/v1'); + +let conditionTypes = { + 'A' : 'AI baseline and on-demand', + 'B' : 'Human baseline and on-demand', + 'C' : 'Only on-demand', + 'D' : 'AI baseline and on-demand (Tutorial)' +} + + +let questionUserStudy4 = { + + addQuestion: function(data, callback) { + // console.log('inside question:', data); + // let questionId = uuidv1(); + let todayDate = new Date(); + let condition = conditionTypes[data.conditionType]; + console.log(data); + console.log({condition}); + const search = "'"; + const replaceWith = "''"; + data.question = data.question.split(search).join(replaceWith); + + const sql = `START TRANSACTION; + INSERT into question_us4 (question_id, question, user_id, video_id, video_timestamp, timestamp, condition_type) VALUES ('${data.questionId}','${data.question}','${data.userId}', '${data.videoId}', ${data.youtubeTimestamp}, '${todayDate.toLocaleString()}', '${condition}'); + COMMIT;`; + // console.log({sql}); + // callback = {questionId: questionId}; + return db.query(sql, callback); + }, + + addCaption: function(data, callback) { + + let captionId = uuidv1(); + let todayDate = new Date(); + let condition = conditionTypes[data.conditionType]; + const sql = `START TRANSACTION; + INSERT into captions_us4 (caption_id, user_id, video_id, caption, keyframe, youtube_timestamp, timestamp, condition_type) VALUES ('${captionId}','${data.userId}', '${data.videoId}', '${data.caption}', '${data.keyFrame}', ${data.youtubeTimestamp}, '${todayDate.toLocaleString()}', '${condition}'); + COMMIT;`; + return db.query(sql, callback); + }, + + addAnswer: function(data, callback) { + + let answerId = uuidv1(); + let todayDate = new Date(); + let isCaption = false; + + const sql = `START TRANSACTION; + INSERT into answer_us4 (answer_id, question_id, user_id, answer, is_caption, keyframes, video_timestamp, timestamp) VALUES ('${answerId}','${data.question_id}','${data.userId}','${data.answer}', ${isCaption}, '${data.key_frame}', ${data.youtube_timestamp}, '${todayDate.toLocaleString()}'); + COMMIT;`; + return db.query(sql, callback); + }, + + /** Adding answer to a particular question*/ + addAnswerToQuestion: function(data, callback) { + + return db.query(`UPDATE question_us4 SET answer = $1 WHERE question_id= $2 AND user_id=$3`, [data.answer, data.questionId, data.userId], callback); + }, + + updateQuestion: function (data, callback) { + // console.log({'IINSIE US': data}); + return db.query(`UPDATE question_us4 SET keyframe = $1, dialog_id = $2 WHERE question_id= $3 AND user_id=$4`, [data.keyframe, data.dialogId, data.questionId, data.userId], callback); + }, + + getQuestionByUserId: function (userId, callback) { + return db.query(`SELECT * FROM question_us4 where user_id = ${userId}`, callback); + }, + + + deleteAllQuestionsByUserId : function(userId, callback){ + // console.log('inside delete'); + return db.query(`Delete from question_us4 where id = '${userId}'`, callback); + const sql = `START TRANSACTION; + Delete from scene_qna_mapping where question_id = '${data.questionId}' AND answer_id = '${data.answerId}' ; + Delete from question where id = '${data.questionId}'; + Delete from answer where id = '${data.answerId}'; + COMMIT;`; + // console.log({sql}); + return db.query(sql, callback); + }, + + addVote: function(data, callback) { + + let voteId = uuidv1(); + let todayDate = new Date(); + let condition = conditionTypes[data.conditionType]; + + const sql = `START TRANSACTION; + INSERT into votes_us4 (vote_id, user_id, video_id, youtube_timestamp, timestamp, is_upvote, condition_type) VALUES ('${voteId}','${data.userId}', '${data.videoId}', ${data.youtubeTimestamp}, '${todayDate.toLocaleString()}', ${data.isUpvote}, '${condition}'); + COMMIT;`; + return db.query(sql, callback); + }, + +}; + + +module.exports = questionUserStudy4; \ No newline at end of file diff --git a/model/scene.js b/model/scene.js index f4faa3c..d3691df 100644 --- a/model/scene.js +++ b/model/scene.js @@ -1,9 +1,15 @@ +/* + contains all the apis related to scene table and merge_scene table +*/ + const db = require('../db/database'); const Utils = require('../backend/utils'); + class Scene { - constructor(scene_id, block_id, scene_num, start_time, end_time, original_description, original_ocr) { + constructor(video_id, scene_id, block_id, scene_num, start_time, end_time, original_description, original_ocr) { + this.video_id = video_id; this.scene_id = scene_id; this.block_id = block_id; this.scene_num = scene_num; @@ -12,7 +18,12 @@ class Scene { this.original_description = original_description; this.original_ocr = original_ocr; } + + static async getSceneData(videoId, callback){ + return db.query(`Select * from scene where video_id = '${videoId}' order by start_time`); + } + setOriginalDescription(desc) { this.original_description = desc; } @@ -21,7 +32,7 @@ class Scene { this.original_ocr = desc; } - static parseSceneJson(sceneJsonObj) { + static parseSceneJson(sceneJsonObj, video_id) { let block_id = '_'; let scene_num = sceneJsonObj.id; let start_time = Utils.parseSecondsFromString(sceneJsonObj.instances[0].start); @@ -29,14 +40,14 @@ class Scene { let original_description = ''; let original_ocr = ''; let scene_id = block_id + '_' + scene_num; - return new Scene(scene_id, block_id, scene_num, start_time, end_time, original_description, original_ocr); + return new Scene(video_id, scene_id, block_id, scene_num, start_time, end_time, original_description, original_ocr); } - static parseSceneArr(sceneJsonArr) { + static parseSceneArr(sceneJsonArr, video_id) { let scenesArr = []; for (let i = 0; i < sceneJsonArr.length; i++) { let sceneJsonObj = sceneJsonArr[i]; - let newScene = Scene.parseSceneJson(sceneJsonObj); + let newScene = Scene.parseSceneJson(sceneJsonObj, video_id); scenesArr.push(newScene); } return scenesArr; @@ -47,7 +58,50 @@ class Scene { try { return await db.tx(t => { var queries = sceneArr.map(scene => { - return t.one('INSERT INTO scene(scene_id, block_id, scene_num, start_time, end_time, original_description, original_ocr) VALUES(${scene_id},${block_id}, ${scene_num}, ${start_time}, ${end_time}, ${original_description}, ${original_ocr}) RETURNING id', scene, a => + a.id); + return t.one('INSERT INTO scene(video_id, scene_id, block_id, scene_num, start_time, end_time, original_description, original_ocr) VALUES(${video_id}, ${scene_id},${block_id}, ${scene_num}, ${start_time}, ${end_time}, ${original_description}, ${original_ocr}) RETURNING id', scene, a => + a.id); + }); + return t.batch(queries); + }); + } catch(e) { + throw e; + } + } + + + static async getSceneDataFromMerge(videoId, callback){ + return db.query(`Select * from merge_scene where video_id = '${videoId}' order by start_time`); + } + + + static async addMergeSceneTable(sceneArr) { + console.log('DB Operation: Inserting Scenes'); + try { + return await db.tx(t => { + var queries = sceneArr.map(scene => { + if(!scene.merged){ + scene.merged = ''; + } + console.log({scene}) + return t.one('INSERT INTO merge_scene(scene_id, block_id, scene_num, start_time, end_time, original_description, original_ocr, video_id, merged) VALUES(${scene_id},${block_id}, ${scene_num}, ${start_time}, ${end_time}, ${original_description}, ${original_ocr},${video_id}, ${merged}) RETURNING id', scene, a => + a.id); + }); + return t.batch(queries); + }); + } catch(e) { + throw e; + } + } + + + static async insertDescriptionTable_blue(sceneArr, userId) { + console.log('DB Operation: Inserting Scenes in Description Table'); + try { + return await db.tx(t => { + var queries = sceneArr.map(scene => { + let descriptionId = userId + '_' + scene.video_id + '_ai_' + scene.scene_num; + return t.one(`INSERT INTO descriptions(desc_id, scene_id, scene_num, start_time, end_time, description, ocr, video_id, volunteer_id, has_ai) + VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, true) RETURNING id`, + [descriptionId, scene.scene_id, scene.scene_num, scene.start_time, scene.end_time, + scene.description_bleu, scene.original_ocr,scene.video_id, userId], a => + a.id); }); return t.batch(queries); }); @@ -75,12 +129,78 @@ class Scene { } } + static async updateBleuDescription_Scene(sceneArr) { + console.log('DB Operation: Updating scene table with description data'); + try { + return await db.tx(t => { + var queries = sceneArr.map(scene => { + return t.one("UPDATE scene SET description_bleu = ${description_bleu} WHERE scene_id = ${scene_id} and video_id = ${video_id} RETURNING id", scene, a => + a.id); + }); + return t.batch(queries); + }); + } catch(e) { + throw e; + } + } + + static async updateBleuPythiaCaption_Scene(sceneArr) { + console.log('DB Operation: Updating scene table with filterd pythia data based on BLEU score'); + try { + return await db.tx(t => { + var queries = sceneArr.map(scene => { + return t.one("UPDATE scene SET pythia_caption_bleu = ${pythia_caption_bleu} WHERE scene_id = ${scene_id} and video_id = ${video_id} RETURNING id", scene, a => + a.id); + }); + return t.batch(queries); + }); + } catch(e) { + throw e; + } + } + + + + static async updateBleuDescription_MergeScene(sceneArr) { + console.log('DB Operation: Updating Merge scene table with description data'); + try { + return await db.tx(t => { + var queries = sceneArr.map(scene => { + return t.one("UPDATE merge_scene SET description_bleu = ${description_bleu} WHERE scene_id = ${scene_id} and video_id = ${video_id} RETURNING id", scene, a => + a.id); + }); + return t.batch(queries); + }); + } catch(e) { + throw e; + } + } + + static async updatePythiaCaptionForAllScenes(sceneArr) { + console.log('DB Operation: Updating Scene table with pythia caption'); + try { + let sql = ''; + sceneArr.forEach((data) => { + sql += `UPDATE scene SET pythia_caption = '${data.caption}' + WHERE scene_id = '${data.sceneId}';`; + + }); + sql = 'START TRANSACTION;' + sql + 'COMMIT;' + console.log({sql}); + + return await db.query(sql); + } catch(e) { + throw e; + } + } + //Do not use static async aggregateShotCaptionsIntoSceneDescription(video) { console.log('DB Operation: Updating Scene table by aggregating description data from Shot table'); return db.any("UPDATE scene SET original_description = (SELECT distinct array_to_string(array_agg(shot.original_description ORDER BY shot.shot_num ASC), E'\n') FROM shot, block, video WHERE scene.scene_id = shot.scene_id AND scene.block_id = block.block_id AND block.video_id = ${youtube_id}) RETURNING id", video, a => + a.id); } + + + + } module.exports = Scene; \ No newline at end of file diff --git a/model/userStudy.js b/model/userStudy.js index 1c5cf7e..0bba7fc 100644 --- a/model/userStudy.js +++ b/model/userStudy.js @@ -45,7 +45,7 @@ let userStudy = { }, deleteSceneById : function(data, videoId, user_id, callback){ - console.log('inside delete', data); + console.log('inside delete', data); let sql = `Delete from descriptions where video_id='${videoId}' and scene_id = '${data.sceneId} and volunteer_id='${user_id}''`; console.log({sql}); return db.query(sql, callback); diff --git a/model/userStudy4.js b/model/userStudy4.js new file mode 100644 index 0000000..a50e5d2 --- /dev/null +++ b/model/userStudy4.js @@ -0,0 +1,63 @@ +const db = require('../db/database'); + +let userStudy4 = { + + addUser: function(user, callback) { + // console.log('Insert user:', user); + return db.query(`INSERT into us_user (uuid,name) VALUES ($1, $2)`, [user.id, user.name], callback); + }, + getAllUsers: function (callback) { + return db.query('SELECT * FROM us_user WHERE number = 5 ORDER BY id ASC', callback); + }, + getUserById: function (id, callback) { + return db.query(`SELECT * FROM us_user where uuid = ${id}`, callback); + }, + getVideoByUserId: function (id, callback) { + return db.query(`SELECT u.user_id, u.video_id, v.title, u.ai_support FROM us_user_video u JOIN video v ON u.video_id = v.youtube_id WHERE u.user_id = ${id}`, callback); + }, + getVideosFromBottom : function(num, callback){ + return db.query(`SELECT * FROM us_user_video ORDER BY id DESC limit ${num}`, callback); + }, + addVideo : function(data, callback){ + return db.query(`INSERT into us_user_video (user_id,youtube_id, ai_support) VALUES ($1, $2, $3)`, [data.id, data.youtube_id, data.ai_support], callback); + }, + getHelperVideo: function (id, callback) { + return db.query(`SELECT u.vi_user_id, u.id, u.volunteer_id, u.video_id, v.title, u.ai_support, u.rating, u.comment, u.video_understanding + FROM us_helper_video u + JOIN video v + ON u.video_id = v.youtube_id + WHERE u.vi_user_id = '${id}' + order by u.id`, callback); + }, + updateStatus_VIUser: function (data, callback) { + console.log({'IINSIE US': data}); + return db.query(`UPDATE us_user_blind set submitted = $1 where uuid= $2`, [data.status, data.id], callback); + }, + updateVIUserInfo: function (data, callback) { + return db.query(`UPDATE us_user_blind set email= $2, vid_desc_frequency=$3, access_preference=$4, access_pref_other= $5 where uuid= $1`, [data.id, data.email, data.vid_desc_frequency, data.access_preference, data.access_pref_other], callback); + }, + getUsers_VIStatus: function (status, callback) { + return db.query(`SELECT * FROM us_user_blind where status= ${status}`, callback); + }, + //get count of users + getAllUser_VI: function (id, callback) { + return db.query(`select * from us_user_blind where uuid = '${id}';`, callback); + }, + + deleteSceneById : function(data, videoId, user_id, callback){ + console.log('inside delete', data); + let sql = `Delete from descriptions where video_id='${videoId}' and scene_id = '${data.sceneId} and volunteer_id='${user_id}''`; + console.log({sql}); + return db.query(sql, callback); + }, + updateSceneEndTime: function (data, callback) { + return db.query(`UPDATE descriptions set end_time = $1 + where video_id = $2 and scene_id= $3 and volunteer_id = $4`, [data.endTime, data.video_id, data.scene_id, data.volunteer_id], callback); + }, + +}; + + +module.exports = userStudy4; + + diff --git a/model/video.js b/model/video.js index afaf829..86cd147 100644 --- a/model/video.js +++ b/model/video.js @@ -14,11 +14,13 @@ class Video { } static parseVideoFromIndexerJson(requestBody) { - let inputJson = JSON.parse(requestBody.inputJson); + // console.log({parseVideoFromIndexerJson : requestBody}); + // console.log({inputJson: requestBody.inputJson}); + let inputJson = requestBody.inputJson; let id = -1; let youtube_id = requestBody.youtubeId; let indexer_video_id = inputJson.id; - let title = requestBody.title; + let title = inputJson.name; let duration = Utils.parseSecondsFromString(inputJson.summarizedInsights.duration.time); let summarised_insights_json = inputJson.summarizedInsights; let insights_json = inputJson.videos[0].insights; @@ -36,6 +38,86 @@ class Video { } } + static async insertIntoVideoAudioTable(newVid) { + console.log('DB Operation: Inserting into video_audio:', newVid); + let query = `INSERT INTO video_audio (youtube_id, video_id, video_title, video_state) VALUES ( $1, $2, $3, $4) + ON CONFLICT(youtube_id) DO UPDATE + SET video_id = $2 , video_title = $3, video_state = $4` + try { + return await db.query(query, [newVid.youtubeId, newVid.video_id, newVid.title, newVid.videoState]); + } catch(e) { + console.log({'error': e}); + throw e; + } + } + + static async fetchVideoAudioTable(newVid) { + try { + return await db.query( `Select * from video_audio where youtube_id= $1`, [newVid.youtubeId]); + } catch(e) { + console.log({'error': e}); + throw e; + } + } + + static async updateVideoAudio_videoState(data) { + console.log('DB Operation: Updating audio_id in table video_audio'); + try { + return await db.query('UPDATE video_audio set video_state=$1 where youtube_id=$2', [data.state, data.youtubeId]); + } catch(e) { + console.log({'error': e}); + throw e; + } + } + + static async updateAudioId_VideoAudio(data) { + console.log('DB Operation: Updating audio_id in table video_audio'); + try { + return await db.query('UPDATE video_audio set audio_id=$1 where youtube_id=$2', [data.audioId, data.youtubeId]); + } catch(e) { + console.log({'error': e}); + throw e; + } + } + + static async updateAudioId(data) { + console.log('DB Operation: Updating Audio'); + try { + return await db.query('UPDATE video set audio_id=$1 where youtube_id=$2', [data.audioId, data.videoId]); + } catch(e) { + console.log({'error': e}); + throw e; + } + } + + static async getSceneByAudioId(audioId) { + console.log('get scene data by audio Id:', audioId); + try { + + let query = `select s.video_id, s.id, s.scene_id, s.block_id, s.scene_num, s.start_time, s.end_time, s.original_description, s.original_ocr + from video v, scene s + where v.youtube_id = s.video_id and v.audio_id = $1 + order by scene_num`; + + return await db.query(query, [audioId]); + } catch(e) { + console.log({'error': e}); + throw e; + } + } + + + static async getVideoAudioByVideoId(videoId) { + console.log('get video data from video_audio table by video Id:', videoId); + try { + return await db.query(`select * from video_audio where video_id = $1`, [videoId]); + } catch(e) { + console.log({'error': e}); + throw e; + } + } + + } module.exports = Video; \ No newline at end of file diff --git a/model/videoDialogue.js b/model/videoDialogue.js new file mode 100644 index 0000000..a644ae5 --- /dev/null +++ b/model/videoDialogue.js @@ -0,0 +1,23 @@ +const db = require('../db/database'); + +let videoDialgue = { + + insertDataIntoVideoDialogue: function(data, videoId, audioId, callback) { + console.log('DB Operation: Inserting timestamps in video_dialogue table'); + console.log({data}); + let sql = ''; + for(let i=0; i< data.length; i++){ + let dialogue = data[i]; + sql += `INSERT INTO video_dialogs(video_id, start_time, end_time, sentence, audio_id) + VALUES('${videoId}', ${dialogue.start}, ${dialogue.end}, '${dialogue.sentence}', '${audioId}');` + } + sql = 'START TRANSACTION;' + sql + 'COMMIT;' ; + console.log({sql}); + return db.query(sql,callback); + } +}; + + +module.exports = videoDialgue; + + diff --git a/notebook.json b/notebook.json new file mode 100644 index 0000000..786e923 --- /dev/null +++ b/notebook.json @@ -0,0 +1 @@ +{"code":0,"msg":"","status":"success","result":{"text":"","sentence":[{"sentence":" Dance with me","confidence":"1.0000","startTime":0,"endTime":0.82},{"sentence":" no!","confidence":"0.0000","startTime":1.14,"endTime":2.19},{"sentence":" Why not?","confidence":"0.6700","startTime":2.27,"endTime":3.34},{"sentence":" Cause I don't want to know","confidence":"0.8400","startTime":3.63,"endTime":6.19},{"sentence":" she's with us","confidence":"1.0000","startTime":6.23,"endTime":8.04},{"sentence":" reality. You wanna ride the ferris with","confidence":"0.7900","startTime":8.16,"endTime":10.24},{"sentence":" feel like","confidence":"0.9600","startTime":10.75,"endTime":11.8},{"sentence":" notice? All right,","confidence":"0.3000","startTime":11.8,"endTime":16.5},{"sentence":" yeah,","confidence":"0.0000","startTime":16.71,"endTime":17.84},{"sentence":" noise","confidence":"0.9600","startTime":19.65,"endTime":20.46},{"sentence":" noise","confidence":"0.9700","startTime":20.85,"endTime":24.56},{"sentence":" what","confidence":"1.0000","startTime":24.56,"endTime":27.44},{"sentence":" noise?","confidence":"0.4900","startTime":27.59,"endTime":29},{"sentence":" Just standing like two inches away from my face. Yeah Thought","confidence":"0.8600","startTime":30.69,"endTime":34.28},{"sentence":" that's no, the","confidence":"0.6800","startTime":34.28,"endTime":36.08},{"sentence":" now I'm christ even came over. I think he likes you","confidence":"0.8800","startTime":36.21,"endTime":40.44},{"sentence":" noise,","confidence":"0.4700","startTime":40.44,"endTime":42.9},{"sentence":" yeah","confidence":"0.7200","startTime":43.39,"endTime":44.16},{"sentence":" red","confidence":"0.3100","startTime":48.33,"endTime":49.06},{"sentence":" noise","confidence":"0.9700","startTime":49.6,"endTime":50.38},{"sentence":" noise","confidence":"0.9100","startTime":50.64,"endTime":52.86},{"sentence":" noise","confidence":"0.9900","startTime":61.07,"endTime":62.26},{"sentence":" yeah","confidence":"0.7100","startTime":62.45,"endTime":64.38},{"sentence":" yeah.","confidence":"0.3500","startTime":64.38,"endTime":65.84},{"sentence":" Hey, what are you doing? You can't do that. Hey, you wanna get down, tommy!","confidence":"0.7000","startTime":65.84,"endTime":71.45},{"sentence":" I'm no care for.","confidence":"0.5900","startTime":71.49,"endTime":72.88},{"sentence":" so","confidence":"1.0000","startTime":72.97,"endTime":74.02},{"sentence":" So it's really nice to who is this guy? No No, cal. And I would really like to take you a friend. Do you mind","confidence":"0.7500","startTime":74.23,"endTime":80.77},{"sentence":" and","confidence":"1.0000","startTime":80.77,"endTime":81.42},{"sentence":" you can't send more to people in a chair. No, okay tell me. All right,","confidence":"0.7600","startTime":81.84,"endTime":86.2},{"sentence":" yeah,","confidence":"0.2700","startTime":87.03,"endTime":87.78},{"sentence":" noise.","confidence":"0.4700","startTime":88.05,"endTime":89.32},{"sentence":" Don know you're gonna kill yourself.","confidence":"0.6600","startTime":89.37,"endTime":92.3},{"sentence":" No! Cut it out!","confidence":"0.6600","startTime":92.3,"endTime":93.84},{"sentence":" There you go.","confidence":"0.6500","startTime":93.84,"endTime":94.95},{"sentence":" What","confidence":"1.0000","startTime":95.14,"endTime":96.36},{"sentence":" no","confidence":"0.0000","startTime":96.55,"endTime":97.88},{"sentence":" no no,","confidence":"0.2800","startTime":98.01,"endTime":99.85},{"sentence":" no. Hay para. She just told you.","confidence":"0.6900","startTime":100.04,"endTime":102.42},{"sentence":" I don't know, because I don't want to.","confidence":"0.7000","startTime":102.42,"endTime":105.14},{"sentence":" Yeah Yeah,","confidence":"0.2800","startTime":105.27,"endTime":106.27},{"sentence":" Right. Well You leave me no choice.","confidence":"0.7600","startTime":106.27,"endTime":108.53},{"sentence":" Oh My God Can we do it? Oh I may ask you one more time.","confidence":"0.6000","startTime":108.81,"endTime":115.71},{"sentence":" Will you?","confidence":"0.6000","startTime":115.94,"endTime":117.02},{"sentence":" We're not go out with me.","confidence":"0.7300","startTime":117.14,"endTime":118.92},{"sentence":" Got there in the head slipping.","confidence":"0.7500","startTime":120,"endTime":122.43},{"sentence":" Not till she greece. I'll go out with him. Honey. Okay Okay Fine. I'll go out with you. I don't do me favors. No No, I want you you want.","confidence":"0.7400","startTime":122.43,"endTime":131.45},{"sentence":" Yes. One. See, I wanna go out with you. You want. Oh,","confidence":"0.5800","startTime":131.45,"endTime":136.93},{"sentence":" all right. Well,","confidence":"0.2000","startTime":137.01,"endTime":138.92},{"sentence":" you think it's so smart? Don't tell anybody. No, you idiot! Oh It's okay. I'll take care of this.","confidence":"0.7100","startTime":139.44,"endTime":146.11},{"sentence":" Wow,","confidence":"0.2200","startTime":146.36,"endTime":147.46},{"sentence":" who are you doing?","confidence":"0.3800","startTime":147.46,"endTime":148.53},{"sentence":" Please don't do that.","confidence":"0.8000","startTime":148.76,"endTime":150},{"sentence":" Don't forget photos.","confidence":"0.7500","startTime":150,"endTime":151.56},{"sentence":" Generally,","confidence":"0.4600","startTime":151.56,"endTime":152.69},{"sentence":" you're not so cocky. Now are yeah,","confidence":"0.7700","startTime":152.87,"endTime":160.45},{"sentence":" maybe we'll maybe will do that.","confidence":"0.5000","startTime":160.51,"endTime":165.81},{"sentence":" like saying oh no. We'll look at that","confidence":"0.7300","startTime":165.81,"endTime":168.08},{"sentence":" at, say, girl from the corner arc.","confidence":"0.6400","startTime":168.55,"endTime":170.77},{"sentence":" Do you remember me?","confidence":"0.8000","startTime":173.19,"endTime":174.4},{"sentence":" Yeah, sure.","confidence":"0.2500","startTime":174.68,"endTime":176.61},{"sentence":" Mister underwear was it. How could I forget? Yeah I wanted to could I don't with you because I'm really sorry about that was really stupid thing to do called fairfield talk to somebody. But I had to be next to you.","confidence":"0.9000","startTime":176.61,"endTime":187.07},{"sentence":" I was being drawn to you.","confidence":"0.8600","startTime":187.44,"endTime":190.24},{"sentence":" Oh Oh Oh Jeez What a line. You use that on. All the girls?","confidence":"0.7600","startTime":190.35,"endTime":197.11},{"sentence":" No,","confidence":"0.4800","startTime":197.52,"endTime":198.39},{"sentence":" right. I saw you give a night with little miss ribbons. What are you doing at what hour tomorrow night or this weekend with it? Why","confidence":"0.7900","startTime":198.57,"endTime":206.33},{"sentence":" why are date? What date did you agree to know? Yes, she did. You promised and use what?","confidence":"0.7300","startTime":206.56,"endTime":212.95},{"sentence":" Well I guess it changed my mind.","confidence":"0.8100","startTime":212.95,"endTime":215.17},{"sentence":" Look I know as you get some dirty guy coming up to you on the street, you don't know him. You don't know me, but but I know me.","confidence":"0.8800","startTime":215.99,"endTime":221.52},{"sentence":" Good Good.","confidence":"0.5900","startTime":222.02,"endTime":223.12},{"sentence":" When I see something, I like","confidence":"0.8600","startTime":223.33,"endTime":225.23},{"sentence":" I got it. I love it.","confidence":"0.7500","startTime":225.49,"endTime":228.48},{"sentence":" Go I mean go crazy for you. What are you talking about?","confidence":"0.8400","startTime":228.48,"endTime":231.52},{"sentence":" Sure,","confidence":"0.4500","startTime":231.75,"endTime":232.56},{"sentence":" Well You","confidence":"1.0000","startTime":232.56,"endTime":233.77},{"sentence":" oh","confidence":"0.9900","startTime":235.16,"endTime":236.08},{"sentence":" you're good.","confidence":"0.6700","startTime":236.08,"endTime":237.1},{"sentence":" What you're good oh good? You're You're fantastic. Really are impressed. I'm not usually like this. And so right,","confidence":"0.5800","startTime":237.1,"endTime":246.78},{"sentence":" I could be fun if you want","confidence":"0.9900","startTime":247.59,"endTime":249.79},{"sentence":" pensive, uh","confidence":"0.6600","startTime":249.79,"endTime":251.73},{"sentence":" smart,","confidence":"0.5000","startTime":251.73,"endTime":253.02},{"sentence":" the superstitious brave.","confidence":"0.7500","startTime":253.29,"endTime":256.27},{"sentence":" And We let them feet.","confidence":"0.8100","startTime":256.44,"endTime":258.56},{"sentence":" I can do whatever you want.","confidence":"0.7000","startTime":258.93,"endTime":260.33},{"sentence":" You just tell me what you want. And i'll be that for","confidence":"0.9200","startTime":261.67,"endTime":263.94},{"sentence":" you're dumb.","confidence":"0.6600","startTime":265.88,"endTime":267.16},{"sentence":" Cool.","confidence":"0.4400","startTime":268.19,"endTime":269.2},{"sentence":" I could be that","confidence":"0.9400","startTime":269.2,"endTime":270.19},{"sentence":" ma one day, what's it gonna hurt?","confidence":"0.5300","startTime":273.41,"endTime":275.88},{"sentence":" I don't think so. What can I do to change your mind?","confidence":"0.7900","startTime":275.88,"endTime":279.41}],"word":[]}} \ No newline at end of file diff --git a/npm_python/nodejs/bleu.js b/npm_python/nodejs/bleu.js new file mode 100644 index 0000000..974c732 --- /dev/null +++ b/npm_python/nodejs/bleu.js @@ -0,0 +1,22 @@ +const {spawn} = require('child_process'); + +async function generateBLEUScore(reference, sentence){ + console.log('nodejs-> ref: ',reference,' and sen: ',sentence); +let dataToSend; + // spawn new child process to call the python script with parameters +const python = spawn('python', ['../python/helloworld.py',reference,sentence]); + // collect data from script + python.stdout.on('data', function (data) { + console.log('Pipe data from python script ...'); + dataToSend = data.toString(); + }); + + python.on('close', (code) => { + console.log(`child process close all stdio with code ${code}`); + return dataToSend + }); +} + +module.exports = { + generateBLEUScore: generateBLEUScore, +} \ No newline at end of file diff --git a/npm_python/nodejs/runpythonscript.js b/npm_python/nodejs/runpythonscript.js new file mode 100644 index 0000000..c46f88b --- /dev/null +++ b/npm_python/nodejs/runpythonscript.js @@ -0,0 +1,26 @@ +const express = require('express') +const {spawn} = require('child_process'); +const app = express() +const port = 3001 +app.get('/', (req, res) => { + + var dataToSend; + // spawn new child process to call the python script +// const python = spawn('python', ['../python/helloworld.py']); +// multiple parameters +const python = spawn('python', ['../python/helloworld.py','this is a test','this is a test']); + // collect data from script + python.stdout.on('data', function (data) { + console.log('Pipe data from python script ...'); + dataToSend = data.toString(); + }); + // in close event we are sure that stream from child process is closed + python.on('close', (code) => { + console.log(`child process close all stdio with code ${code}`); + // send data to browser + res.send(dataToSend) + }); + +}) +app.listen(port, () => console.log(`Example app listening on port +${port}!`)) \ No newline at end of file diff --git a/npm_python/package-lock.json b/npm_python/package-lock.json new file mode 100644 index 0000000..90513dc --- /dev/null +++ b/npm_python/package-lock.json @@ -0,0 +1,374 @@ +{ + "name": "npm_python", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "accepts": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", + "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==", + "requires": { + "mime-types": "~2.1.24", + "negotiator": "0.6.2" + } + }, + "array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + }, + "body-parser": { + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", + "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", + "requires": { + "bytes": "3.1.0", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "1.7.2", + "iconv-lite": "0.4.24", + "on-finished": "~2.3.0", + "qs": "6.7.0", + "raw-body": "2.4.0", + "type-is": "~1.6.17" + } + }, + "bytes": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", + "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==" + }, + "content-disposition": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", + "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==", + "requires": { + "safe-buffer": "5.1.2" + } + }, + "content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" + }, + "cookie": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", + "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==" + }, + "cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + }, + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" + }, + "destroy": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + }, + "encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + }, + "escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + }, + "etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" + }, + "express": { + "version": "4.17.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", + "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", + "requires": { + "accepts": "~1.3.7", + "array-flatten": "1.1.1", + "body-parser": "1.19.0", + "content-disposition": "0.5.3", + "content-type": "~1.0.4", + "cookie": "0.4.0", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.1.2", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.5", + "qs": "6.7.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.1.2", + "send": "0.17.1", + "serve-static": "1.14.1", + "setprototypeof": "1.1.1", + "statuses": "~1.5.0", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + } + }, + "finalhandler": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", + "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "requires": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "statuses": "~1.5.0", + "unpipe": "~1.0.0" + } + }, + "forwarded": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", + "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" + }, + "fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" + }, + "http-errors": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", + "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.1", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.0" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" + }, + "media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + }, + "merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + }, + "methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" + }, + "mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" + }, + "mime-db": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.43.0.tgz", + "integrity": "sha512-+5dsGEEovYbT8UY9yD7eE4XTc4UwJ1jBYlgaQQF38ENsKR3wj/8q8RFZrF9WIZpB2V1ArTVFUva8sAul1NzRzQ==" + }, + "mime-types": { + "version": "2.1.26", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.26.tgz", + "integrity": "sha512-01paPWYgLrkqAyrlDorC1uDwl2p3qZT7yl806vW7DvDoxwXi46jsjFbg+WdwotBIk6/MbEhO/dh5aZ5sNj/dWQ==", + "requires": { + "mime-db": "1.43.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "negotiator": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", + "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" + }, + "on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "requires": { + "ee-first": "1.1.1" + } + }, + "parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" + }, + "path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + }, + "proxy-addr": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", + "integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==", + "requires": { + "forwarded": "~0.1.2", + "ipaddr.js": "1.9.1" + } + }, + "qs": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", + "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==" + }, + "range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" + }, + "raw-body": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", + "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==", + "requires": { + "bytes": "3.1.0", + "http-errors": "1.7.2", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "send": { + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", + "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==", + "requires": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "~1.7.2", + "mime": "1.6.0", + "ms": "2.1.1", + "on-finished": "~2.3.0", + "range-parser": "~1.2.1", + "statuses": "~1.5.0" + }, + "dependencies": { + "ms": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" + } + } + }, + "serve-static": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", + "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==", + "requires": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.17.1" + } + }, + "setprototypeof": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", + "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" + }, + "statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" + }, + "toidentifier": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", + "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" + }, + "type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + } + }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + }, + "utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + }, + "vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + } + } +} diff --git a/npm_python/package.json b/npm_python/package.json new file mode 100644 index 0000000..bba0173 --- /dev/null +++ b/npm_python/package.json @@ -0,0 +1,14 @@ +{ + "name": "npm_python", + "version": "1.0.0", + "description": "Run python in nodejs", + "main": "index.js", + "scripts": { + "test": "---" + }, + "author": "", + "license": "ISC", + "dependencies": { + "express": "^4.17.1" + } +} diff --git a/npm_python/python/helloworld.py b/npm_python/python/helloworld.py new file mode 100644 index 0000000..2b2e4d9 --- /dev/null +++ b/npm_python/python/helloworld.py @@ -0,0 +1,14 @@ +import sys + +from nltk.translate.bleu_score import sentence_bleu + +print('#Hello from python#') + +sentence1 = sys.argv[1] +sentence2 = sys.argv[2] +sentence1 = sentence1.split(' ') +sentence2 = sentence2.split(' ') + +reference = [sentence1] +score = sentence_bleu(reference, sentence2) +print('## The BLEU score is: ',score) diff --git a/oceans8.json b/oceans8.json new file mode 100644 index 0000000..19a0227 --- /dev/null +++ b/oceans8.json @@ -0,0 +1 @@ +{"code":0,"msg":"","status":"success","result":{"text":"","sentence":[{"sentence":" Noise","confidence":"0.8300","startTime":23.25,"endTime":24.68},{"sentence":" noise","confidence":"0.9700","startTime":24.82,"endTime":27},{"sentence":" noise.","confidence":"0.4900","startTime":27,"endTime":29.56},{"sentence":" Hi, I like to return these. Oh Of Course Do You Have you ever seen? No, but there unopened, they haven't touched. I really need to receipt.","confidence":"0.7300","startTime":29.65,"endTime":38.22},{"sentence":" They are sealed their brand new. Do you have the credit card that you use? This is ridiculous. I I bought these last week. Maybe you can try client services on the 6th floor.","confidence":"0.8600","startTime":38.24,"endTime":47.51},{"sentence":" Never mind just keep them. Jesus","confidence":"0.8600","startTime":47.56,"endTime":50.69},{"sentence":" can at least get a bag. Sure,","confidence":"0.7800","startTime":50.91,"endTime":53.32},{"sentence":" yeah,","confidence":"0.3900","startTime":53.88,"endTime":55},{"sentence":" thank you for","confidence":"0.9400","startTime":55.4,"endTime":57.67},{"sentence":" I'm checking out in 2014, gary randall did you know, pleasant stay terrific.","confidence":"0.7200","startTime":68.17,"endTime":73.25},{"sentence":" Okay, my name is monica. There anything else I can do for you? Or arrange transportation? The girl said,","confidence":"0.7900","startTime":74.5,"endTime":79.72},{"sentence":" hi, this is mrs. Randall. We just checked out of room 2084. May I speak with monica? Please? Is she? Hi, monica! Hi Hi. Uh We just found out our flight was cancelled. No I no no,","confidence":"0.6800","startTime":79.91,"endTime":93.99},{"sentence":" And Instead of saying some horrible airport hotel, we were hoping that maybe we could get our original room back. Yeah We can give you that same room back.","confidence":"0.9100","startTime":93.99,"endTime":101.64},{"sentence":" That would be amazing. Thank you so much. Um We're just gonna go grab a bite. Uh Would it be possible to get the maid in there now?","confidence":"0.8600","startTime":101.64,"endTime":108.9},{"sentence":" Yes, absolutely perfect. We really appreciate it.","confidence":"0.7000","startTime":108.9,"endTime":112.35},{"sentence":" It's not a problem. Thank you so much. We'll see you soon.","confidence":"0.8000","startTime":112.35,"endTime":115.22},{"sentence":" Hi,","confidence":"0.3800","startTime":115.22,"endTime":116.44},{"sentence":" hi,","confidence":"0.5000","startTime":126.59,"endTime":127.6},{"sentence":" I'm so sorry. Can you finish up later? I just have to get off my feet. Of Course Thank you so much. Have a great day! Bye bye.","confidence":"0.7800","startTime":127.6,"endTime":134.41}],"word":[]}} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 512925f..9cfc2d0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,29 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@babel/code-frame": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.8.3.tgz", + "integrity": "sha512-a9gxpmdXtZEInkCSHUJDLHZVBgb1QS0jhss4cPP93EW7s+uC5bikET2twEF3KV+7rDblJcmNvTR7VJejqd2C2g==", + "requires": { + "@babel/highlight": "^7.8.3" + } + }, + "@babel/helper-validator-identifier": { + "version": "7.9.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.9.5.tgz", + "integrity": "sha512-/8arLKUFq882w4tWGj9JYzRpAlZgiWUJ+dtteNTDqrRBz9Iguck9Rn3ykuBDoUwh2TO4tSAJlrxDUOXWklJe4g==" + }, + "@babel/highlight": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.9.0.tgz", + "integrity": "sha512-lJZPilxX7Op3Nv/2cvFdnlepPXDxi29wxteT57Q965oc5R9v86ztx0jfxVrTcBk8C2kcPkkDa2Z4T3ZsPPVWsQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.9.0", + "chalk": "^2.0.0", + "js-tokens": "^4.0.0" + } + }, "@google-cloud/common": { "version": "0.32.1", "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.32.1.tgz", @@ -278,9 +301,9 @@ "integrity": "sha512-UFbtbGPjstz0eWHb+ga/GM3Z9EzqKXFWIbSOFURU0A/Gku0Bky4bCk9/h//K2Xr3IrCfjFNhMm4jyZ5dbCewGA==" }, "@hapi/hoek": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-8.3.2.tgz", - "integrity": "sha512-NP5SG4bzix+EtSMtcudp8TvI0lB46mXNo8uFpTDw6tqxGx4z5yx+giIunEFA0Z7oUO4DuWrOJV9xqR2tJVEdyA==" + "version": "8.5.1", + "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-8.5.1.tgz", + "integrity": "sha512-yN7kbciD87WzLGc5539Tn0sApjyiGHAJgKvG9W8C7O+6c7qmoQMfVs0W4bX17eqz6C78QJqqFrtgdK5EWf6Qow==" }, "@hapi/joi": { "version": "16.1.7", @@ -361,6 +384,15 @@ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=" }, + "@types/bytebuffer": { + "version": "5.0.40", + "resolved": "https://registry.npmjs.org/@types/bytebuffer/-/bytebuffer-5.0.40.tgz", + "integrity": "sha512-h48dyzZrPMz25K6Q4+NCwWaxwXany2FhQg/ErOcdZS1ZpsaDnDMZg8JYLMTGz7uvXKrcKGJUZJlZObyfgdaN9g==", + "requires": { + "@types/long": "*", + "@types/node": "*" + } + }, "@types/caseless": { "version": "0.12.2", "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", @@ -419,6 +451,16 @@ "negotiator": "0.6.2" } }, + "acorn": { + "version": "6.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", + "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==" + }, + "acorn-jsx": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.2.0.tgz", + "integrity": "sha512-HiUX/+K2YpkpJ+SzBffkM/AQ2YE03S0U1kjTLVpoJdhZMOWy8qvXVN9JdLqv2QsaQ6MPYQIuNmwD8zOiYUofLQ==" + }, "after": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/after/-/after-0.8.2.tgz", @@ -433,16 +475,21 @@ } }, "ajv": { - "version": "6.10.2", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.2.tgz", - "integrity": "sha512-TXtUUEYHuaTEbLZWIKUr5pmBuhDLy+8KYtPYdcV8qC+pOZL+NKqYwvWSRrVXHn+ZmRRAu8vJTAznH7Oag6RVRw==", + "version": "6.12.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.0.tgz", + "integrity": "sha512-D6gFiFA0RRLyUbvijN74DWAjXSFxWKaWP7mldxkVhyhAV3+SWA9HEJPHQ2c9soIeTFJqcSdFDGFgdqs1iUU2Hw==", "requires": { - "fast-deep-equal": "^2.0.1", + "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" } }, + "amdefine": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", + "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=" + }, "ansi-align": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-2.0.0.tgz", @@ -480,6 +527,11 @@ } } }, + "ansi-escapes": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", + "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==" + }, "ansi-regex": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", @@ -517,6 +569,11 @@ "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", "integrity": "sha1-HjRA6RXwsSA9I3SOeO3XubW0PlY=" }, + "aproba": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==" + }, "archiver": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/archiver/-/archiver-3.1.1.tgz", @@ -568,6 +625,23 @@ "readable-stream": "^2.0.0" } }, + "are-we-there-yet": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", + "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", + "requires": { + "delegates": "^1.0.0", + "readable-stream": "^2.0.6" + } + }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "requires": { + "sprintf-js": "~1.0.2" + } + }, "arr-diff": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz", @@ -641,6 +715,11 @@ "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz", "integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c=" }, + "astral-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz", + "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==" + }, "async": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/async/-/async-2.6.2.tgz", @@ -675,9 +754,9 @@ "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" }, "aws4": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", - "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.9.1.tgz", + "integrity": "sha512-wMHVg2EOHaMRxbzgFJ9gtjOOCrI80OHLG14rxi28XwOW8ux6IiEbRCGGGqCtdAIg4FQCbW20k9RsT4y3gJlFug==" }, "axios": { "version": "0.18.1", @@ -688,6 +767,11 @@ "is-buffer": "^2.0.2" } }, + "axo": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/axo/-/axo-0.0.2.tgz", + "integrity": "sha1-STVfu+qzhEm8ppahqsxGml7p/Uc=" + }, "backo2": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/backo2/-/backo2-1.0.2.tgz", @@ -811,6 +895,15 @@ "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.13.1.tgz", "integrity": "sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw==" }, + "bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "optional": true, + "requires": { + "file-uri-to-path": "1.0.0" + } + }, "biskviit": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/biskviit/-/biskviit-1.0.1.tgz", @@ -820,17 +913,17 @@ } }, "bl": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-3.0.0.tgz", - "integrity": "sha512-EUAyP5UHU5hxF8BPT0LKW8gjYLhq1DQIcneOX/pL/m2Alo+OYDQAJlHq+yseMP50Os2nHXOSic6Ss3vSQeyf4A==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/bl/-/bl-3.0.1.tgz", + "integrity": "sha512-jrCW5ZhfQ/Vt07WX1Ngs+yn9BDqPL/gw28S7s9H6QK/gupnizNzJAss5akW20ISgOrbLTlXOOCTJeNUQqruAWQ==", "requires": { "readable-stream": "^3.0.1" }, "dependencies": { "readable-stream": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz", - "integrity": "sha512-jItXPLmrSR8jmTRmRWJXCnGJsfy85mB3Wd/uINMXA65yrnFo0cPClFIUWzo2najVNSl+mx7/4W8ttlLWJe99pQ==", + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", "requires": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -850,20 +943,57 @@ "integrity": "sha512-5am6HnnfN+urzt4yfg7IgTbotDjIT/u8AJpEt0sIU9FtXfVeezXAPKswrG+xKUCOYAINpSdgZVDU6QFh+cuH3w==" }, "body-parser": { - "version": "1.18.3", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz", - "integrity": "sha1-WykhmP/dVTs6DyDe0FkrlWlVyLQ=", + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", + "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", "requires": { - "bytes": "3.0.0", + "bytes": "3.1.0", "content-type": "~1.0.4", "debug": "2.6.9", "depd": "~1.1.2", - "http-errors": "~1.6.3", - "iconv-lite": "0.4.23", + "http-errors": "1.7.2", + "iconv-lite": "0.4.24", "on-finished": "~2.3.0", - "qs": "6.5.2", - "raw-body": "2.3.3", - "type-is": "~1.6.16" + "qs": "6.7.0", + "raw-body": "2.4.0", + "type-is": "~1.6.17" + }, + "dependencies": { + "http-errors": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", + "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.1", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.0" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "qs": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", + "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==" + }, + "setprototypeof": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", + "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" + }, + "statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" + } } }, "boxen": { @@ -1062,9 +1192,9 @@ } }, "bytes": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", - "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", + "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==" }, "cache-base": { "version": "1.0.1", @@ -1087,6 +1217,11 @@ "resolved": "https://registry.npmjs.org/callsite/-/callsite-1.0.0.tgz", "integrity": "sha1-KAOY5dZkvXQDi28JBRU+borxvCA=" }, + "callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==" + }, "camelcase": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-2.1.1.tgz", @@ -1134,16 +1269,26 @@ "supports-color": "^5.3.0" } }, + "chardet": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", + "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==" + }, "check-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", "dev": true }, + "child_process": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/child_process/-/child_process-1.0.2.tgz", + "integrity": "sha1-sffn/HPSXn/R1FWtyU4UODAYK1o=" + }, "chokidar": { - "version": "2.1.6", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.1.6.tgz", - "integrity": "sha512-V2jUo67OKkc6ySiRpJrjlpJKl9kDuG+Xb8VgsGzb+aEouhgS1D0weyPU4lEzdAcsCAvrih2J2BqyXqHWvVLw5g==", + "version": "2.1.8", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.1.8.tgz", + "integrity": "sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg==", "requires": { "anymatch": "^2.0.0", "async-each": "^1.0.1", @@ -1159,6 +1304,11 @@ "upath": "^1.1.1" } }, + "chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, "ci-info": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-1.6.0.tgz", @@ -1195,6 +1345,57 @@ "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-1.0.0.tgz", "integrity": "sha1-T6kXw+WclKAEzWH47lCdplFocUM=" }, + "cli-cursor": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", + "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=", + "requires": { + "restore-cursor": "^2.0.0" + } + }, + "cli-progress": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-2.1.1.tgz", + "integrity": "sha512-TSJw3LY9ZRSis7yYzQ7flIdtQMbacd9oYoiFphJhI4SzgmqF0zErO+uNv0lbUjk1L4AGfHQJ4OVYYzW+JV66KA==", + "requires": { + "colors": "^1.1.2", + "string-width": "^2.1.1" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=" + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "requires": { + "ansi-regex": "^3.0.0" + } + } + } + }, + "cli-width": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.1.tgz", + "integrity": "sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw==" + }, "cliui": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz", @@ -1232,6 +1433,11 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" }, + "colors": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", + "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==" + }, "colour": { "version": "0.7.1", "resolved": "https://registry.npmjs.org/colour/-/colour-0.7.1.tgz", @@ -1325,6 +1531,11 @@ "xdg-basedir": "^3.0.0" } }, + "console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=" + }, "content-disposition": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", @@ -1364,6 +1575,15 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, + "cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "requires": { + "object-assign": "^4", + "vary": "^1" + } + }, "crc": { "version": "3.8.0", "resolved": "https://registry.npmjs.org/crc/-/crc-3.8.0.tgz", @@ -1432,9 +1652,9 @@ }, "dependencies": { "minimist": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", - "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" } } }, @@ -1488,6 +1708,11 @@ "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" }, + "deep-is": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", + "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=" + }, "define-property": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz", @@ -1530,6 +1755,11 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, + "delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" + }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -1540,6 +1770,11 @@ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" }, + "detect-libc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", + "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=" + }, "dicer": { "version": "0.2.5", "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.2.5.tgz", @@ -1578,10 +1813,18 @@ "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", "dev": true }, + "doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "requires": { + "esutils": "^2.0.2" + } + }, "dot-prop": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.0.tgz", - "integrity": "sha512-tUMXrxlExSW6U2EXiiKGSBVdYgtV8qlHL+C10TsW4PURY/ic+eaysnSkwB4kA/mBlCyy/IKDJ+Lc3wbWeaXtuQ==", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.1.tgz", + "integrity": "sha512-l0p4+mIuJIua0mhxGoh4a+iNL9bmeK5DvnSVQa6T0OhrVmaEa1XScX5Etc673FePCJOArq/4Pa2cLGODUWTPOQ==", "requires": { "is-obj": "^1.0.0" } @@ -1777,6 +2020,147 @@ "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" }, + "eslint": { + "version": "5.16.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.16.0.tgz", + "integrity": "sha512-S3Rz11i7c8AA5JPv7xAH+dOyq/Cu/VXHiHXBPOU1k/JAM5dXqQPt3qcrhpHSorXmrpu2g0gkIBVXAqCpzfoZIg==", + "requires": { + "@babel/code-frame": "^7.0.0", + "ajv": "^6.9.1", + "chalk": "^2.1.0", + "cross-spawn": "^6.0.5", + "debug": "^4.0.1", + "doctrine": "^3.0.0", + "eslint-scope": "^4.0.3", + "eslint-utils": "^1.3.1", + "eslint-visitor-keys": "^1.0.0", + "espree": "^5.0.1", + "esquery": "^1.0.1", + "esutils": "^2.0.2", + "file-entry-cache": "^5.0.1", + "functional-red-black-tree": "^1.0.1", + "glob": "^7.1.2", + "globals": "^11.7.0", + "ignore": "^4.0.6", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "inquirer": "^6.2.2", + "js-yaml": "^3.13.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.3.0", + "lodash": "^4.17.11", + "minimatch": "^3.0.4", + "mkdirp": "^0.5.1", + "natural-compare": "^1.4.0", + "optionator": "^0.8.2", + "path-is-inside": "^1.0.2", + "progress": "^2.0.0", + "regexpp": "^2.0.1", + "semver": "^5.5.1", + "strip-ansi": "^4.0.0", + "strip-json-comments": "^2.0.1", + "table": "^5.2.3", + "text-table": "^0.2.0" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=" + }, + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "requires": { + "ansi-regex": "^3.0.0" + } + } + } + }, + "eslint-scope": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", + "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", + "requires": { + "esrecurse": "^4.1.0", + "estraverse": "^4.1.1" + } + }, + "eslint-utils": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-1.4.3.tgz", + "integrity": "sha512-fbBN5W2xdY45KulGXmLHZ3c3FHfVYmKg0IrAKGOkT/464PQsx2UeIzfz1RmEci+KLm1bBaAzZAh8+/E+XAeZ8Q==", + "requires": { + "eslint-visitor-keys": "^1.1.0" + } + }, + "eslint-visitor-keys": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.1.0.tgz", + "integrity": "sha512-8y9YjtM1JBJU/A9Kc+SbaOV4y29sSWckBwMHa+FGtVj5gN/sbnKDf6xJUl+8g7FAij9LVaP8C24DUiH/f/2Z9A==" + }, + "espree": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-5.0.1.tgz", + "integrity": "sha512-qWAZcWh4XE/RwzLJejfcofscgMc9CamR6Tn1+XRXNzrvUSSbiAjGOI/fggztjIi7y9VLPqnICMIPiGyr8JaZ0A==", + "requires": { + "acorn": "^6.0.7", + "acorn-jsx": "^5.0.0", + "eslint-visitor-keys": "^1.0.0" + } + }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" + }, + "esquery": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.3.1.tgz", + "integrity": "sha512-olpvt9QG0vniUBZspVRN6lwB7hOZoTRtT+jzR+tS4ffYx2mzbw+z0XCOk44aaLYKApNX5nMm+E+P6o25ip/DHQ==", + "requires": { + "estraverse": "^5.1.0" + }, + "dependencies": { + "estraverse": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.1.0.tgz", + "integrity": "sha512-FyohXK+R0vE+y1nHLoBM7ZTyqRpqAlhdZHCWIWEviFLiGB8b04H6bQs8G+XTthacvT8VuwvteiP7RJSxMs8UEw==" + } + } + }, + "esrecurse": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz", + "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==", + "requires": { + "estraverse": "^4.1.0" + } + }, + "estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==" + }, + "esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==" + }, "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", @@ -1787,6 +2171,11 @@ "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" }, + "eventemitter3": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.0.tgz", + "integrity": "sha512-qerSRB0p+UDEssxTtm6EDKcE7W4OaoisfIMl4CngyEhjpYglocpNg6UEqCvemdGhosAsg4sO2dXJOdyBifPGCg==" + }, "exceljs": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/exceljs/-/exceljs-3.3.1.tgz", @@ -1901,6 +2290,46 @@ "type-is": "~1.6.16", "utils-merge": "1.0.1", "vary": "~1.1.2" + }, + "dependencies": { + "body-parser": { + "version": "1.18.3", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz", + "integrity": "sha1-WykhmP/dVTs6DyDe0FkrlWlVyLQ=", + "requires": { + "bytes": "3.0.0", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "~1.6.3", + "iconv-lite": "0.4.23", + "on-finished": "~2.3.0", + "qs": "6.5.2", + "raw-body": "2.3.3", + "type-is": "~1.6.16" + } + }, + "bytes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", + "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" + }, + "qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + }, + "raw-body": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.3.tgz", + "integrity": "sha512-9esiElv1BrZoI3rCDuOuKCBRbuApGGaDPQfjSflGxdy4oyzqghxu6klEkkVIvBje+FF0BX9coEv8KqW6X/7njw==", + "requires": { + "bytes": "3.0.0", + "http-errors": "1.6.3", + "iconv-lite": "0.4.23", + "unpipe": "1.0.0" + } + } } }, "extend": { @@ -1927,6 +2356,39 @@ } } }, + "extendible": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/extendible/-/extendible-0.1.1.tgz", + "integrity": "sha1-4qN+2HEp+0+VM+io11BiMKU5yQU=" + }, + "external-editor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", + "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==", + "requires": { + "chardet": "^0.7.0", + "iconv-lite": "^0.4.24", + "tmp": "^0.0.33" + }, + "dependencies": { + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "tmp": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", + "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "requires": { + "os-tmpdir": "~1.0.2" + } + } + } + }, "extglob": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz", @@ -1991,6 +2453,11 @@ "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" }, + "failure": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/failure/-/failure-1.1.1.tgz", + "integrity": "sha1-qOg9OxYC0kaL/2rU2QceAQO4Goc=" + }, "fast-csv": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-3.4.0.tgz", @@ -2002,14 +2469,19 @@ } }, "fast-deep-equal": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", - "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.1.tgz", + "integrity": "sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA==" }, "fast-json-stable-stringify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", - "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" + }, + "fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" }, "fast-text-encoding": { "version": "1.0.0", @@ -2025,11 +2497,41 @@ "encoding": "0.1.12" } }, + "ffmpeg": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/ffmpeg/-/ffmpeg-0.0.4.tgz", + "integrity": "sha1-HEYN+OfaUSf2LO70v6BsWciWMMs=", + "requires": { + "when": ">= 0.0.1" + } + }, "ffprobe-static": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/ffprobe-static/-/ffprobe-static-3.0.0.tgz", "integrity": "sha512-LZ1Sj4RHS95t/ReZtRbCmMEWDDl7cfIMwmhNgCZcdtOgRlAHaGGOupoc166Q9AV+T7lXnUvu5HYczcsn69c6fw==" }, + "figures": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", + "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=", + "requires": { + "escape-string-regexp": "^1.0.5" + } + }, + "file-entry-cache": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-5.0.1.tgz", + "integrity": "sha512-bCg29ictuBaKUwwArK4ouCaqDgLZcysCFLmM/Yn/FDoqndh/9vNuQfXRDvTuXKLxfD/JtZQGKFT8MGcJBK644g==", + "requires": { + "flat-cache": "^2.0.1" + } + }, + "file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "optional": true + }, "fill-range": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", @@ -2073,6 +2575,31 @@ "locate-path": "^3.0.0" } }, + "flat-cache": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-2.0.1.tgz", + "integrity": "sha512-LoQe6yDuUMDzQAEH8sgmh4Md6oZnc/7PjtwjNFSzveXqSHt6ka9fPBuso7IGf9Rz4uqnSnWiFH2B/zj24a5ReA==", + "requires": { + "flatted": "^2.0.0", + "rimraf": "2.6.3", + "write": "1.0.3" + }, + "dependencies": { + "rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "requires": { + "glob": "^7.1.3" + } + } + } + }, + "flatted": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", + "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==" + }, "fluent-ffmpeg": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz", @@ -2158,19 +2685,28 @@ "universalify": "^0.1.0" } }, + "fs-minipass": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz", + "integrity": "sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==", + "requires": { + "minipass": "^2.6.0" + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, "fsevents": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.9.tgz", - "integrity": "sha512-oeyj2H3EjjonWcFjD5NvZNE9Rqe4UW+nQBU2HNeKw0koVLEFIhtyETyAakeAM3de7Z/SW5kcA+fZUait9EApnw==", + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.12.tgz", + "integrity": "sha512-Ggd/Ktt7E7I8pxZRbGIs7vwqAPscSESMrCSkx2FtWeqmheJgCo2R74fTsZFCifr0VTPwqRpPv17+6b8Zp7th0Q==", "optional": true, "requires": { + "bindings": "^1.5.0", "nan": "^2.12.1", - "node-pre-gyp": "^0.12.0" + "node-pre-gyp": "*" }, "dependencies": { "abbrev": { @@ -2180,8 +2716,7 @@ }, "ansi-regex": { "version": "2.1.1", - "bundled": true, - "optional": true + "bundled": true }, "aproba": { "version": "1.2.0", @@ -2199,37 +2734,32 @@ }, "balanced-match": { "version": "1.0.0", - "bundled": true, - "optional": true + "bundled": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, - "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "chownr": { - "version": "1.1.1", + "version": "1.1.4", "bundled": true, "optional": true }, "code-point-at": { "version": "1.1.0", - "bundled": true, - "optional": true + "bundled": true }, "concat-map": { "version": "0.0.1", - "bundled": true, - "optional": true + "bundled": true }, "console-control-strings": { "version": "1.1.0", - "bundled": true, - "optional": true + "bundled": true }, "core-util-is": { "version": "1.0.2", @@ -2237,7 +2767,7 @@ "optional": true }, "debug": { - "version": "4.1.1", + "version": "3.2.6", "bundled": true, "optional": true, "requires": { @@ -2260,11 +2790,11 @@ "optional": true }, "fs-minipass": { - "version": "1.2.5", + "version": "1.2.7", "bundled": true, "optional": true, "requires": { - "minipass": "^2.2.1" + "minipass": "^2.6.0" } }, "fs.realpath": { @@ -2288,7 +2818,7 @@ } }, "glob": { - "version": "7.1.3", + "version": "7.1.6", "bundled": true, "optional": true, "requires": { @@ -2314,7 +2844,7 @@ } }, "ignore-walk": { - "version": "3.0.1", + "version": "3.0.3", "bundled": true, "optional": true, "requires": { @@ -2331,9 +2861,8 @@ } }, "inherits": { - "version": "2.0.3", - "bundled": true, - "optional": true + "version": "2.0.4", + "bundled": true }, "ini": { "version": "1.3.5", @@ -2343,7 +2872,6 @@ "is-fullwidth-code-point": { "version": "1.0.0", "bundled": true, - "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -2356,58 +2884,54 @@ "minimatch": { "version": "3.0.4", "bundled": true, - "optional": true, "requires": { "brace-expansion": "^1.1.7" } }, "minimist": { - "version": "0.0.8", - "bundled": true, - "optional": true + "version": "1.2.5", + "bundled": true }, "minipass": { - "version": "2.3.5", + "version": "2.9.0", "bundled": true, - "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" } }, "minizlib": { - "version": "1.2.1", + "version": "1.3.3", "bundled": true, "optional": true, "requires": { - "minipass": "^2.2.1" + "minipass": "^2.9.0" } }, "mkdirp": { - "version": "0.5.1", + "version": "0.5.3", "bundled": true, - "optional": true, "requires": { - "minimist": "0.0.8" + "minimist": "^1.2.5" } }, "ms": { - "version": "2.1.1", + "version": "2.1.2", "bundled": true, "optional": true }, "needle": { - "version": "2.3.0", + "version": "2.3.3", "bundled": true, "optional": true, "requires": { - "debug": "^4.1.0", + "debug": "^3.2.6", "iconv-lite": "^0.4.4", "sax": "^1.2.4" } }, "node-pre-gyp": { - "version": "0.12.0", + "version": "0.14.0", "bundled": true, "optional": true, "requires": { @@ -2420,11 +2944,11 @@ "rc": "^1.2.7", "rimraf": "^2.6.1", "semver": "^5.3.0", - "tar": "^4" + "tar": "^4.4.2" } }, "nopt": { - "version": "4.0.1", + "version": "4.0.3", "bundled": true, "optional": true, "requires": { @@ -2433,17 +2957,25 @@ } }, "npm-bundled": { - "version": "1.0.6", + "version": "1.1.1", "bundled": true, - "optional": true + "optional": true, + "requires": { + "npm-normalize-package-bin": "^1.0.1" + } + }, + "npm-normalize-package-bin": { + "version": "1.0.1", + "bundled": true }, "npm-packlist": { - "version": "1.4.1", + "version": "1.4.8", "bundled": true, "optional": true, "requires": { "ignore-walk": "^3.0.1", - "npm-bundled": "^1.0.1" + "npm-bundled": "^1.0.1", + "npm-normalize-package-bin": "^1.0.1" } }, "npmlog": { @@ -2459,8 +2991,7 @@ }, "number-is-nan": { "version": "1.0.1", - "bundled": true, - "optional": true + "bundled": true }, "object-assign": { "version": "4.1.1", @@ -2470,7 +3001,6 @@ "once": { "version": "1.4.0", "bundled": true, - "optional": true, "requires": { "wrappy": "1" } @@ -2500,7 +3030,7 @@ "optional": true }, "process-nextick-args": { - "version": "2.0.0", + "version": "2.0.1", "bundled": true, "optional": true }, @@ -2513,17 +3043,10 @@ "ini": "~1.3.0", "minimist": "^1.2.0", "strip-json-comments": "~2.0.1" - }, - "dependencies": { - "minimist": { - "version": "1.2.0", - "bundled": true, - "optional": true - } } }, "readable-stream": { - "version": "2.3.6", + "version": "2.3.7", "bundled": true, "optional": true, "requires": { @@ -2537,7 +3060,7 @@ } }, "rimraf": { - "version": "2.6.3", + "version": "2.7.1", "bundled": true, "optional": true, "requires": { @@ -2546,8 +3069,7 @@ }, "safe-buffer": { "version": "5.1.2", - "bundled": true, - "optional": true + "bundled": true }, "safer-buffer": { "version": "2.1.2", @@ -2560,7 +3082,7 @@ "optional": true }, "semver": { - "version": "5.7.0", + "version": "5.7.1", "bundled": true, "optional": true }, @@ -2577,7 +3099,6 @@ "string-width": { "version": "1.0.2", "bundled": true, - "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -2595,7 +3116,6 @@ "strip-ansi": { "version": "3.0.1", "bundled": true, - "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -2606,17 +3126,17 @@ "optional": true }, "tar": { - "version": "4.4.8", + "version": "4.4.13", "bundled": true, "optional": true, "requires": { "chownr": "^1.1.1", "fs-minipass": "^1.2.5", - "minipass": "^2.3.4", - "minizlib": "^1.1.1", + "minipass": "^2.8.6", + "minizlib": "^1.2.1", "mkdirp": "^0.5.0", "safe-buffer": "^5.1.2", - "yallist": "^3.0.2" + "yallist": "^3.0.3" } }, "util-deprecate": { @@ -2634,13 +3154,11 @@ }, "wrappy": { "version": "1.0.2", - "bundled": true, - "optional": true + "bundled": true }, "yallist": { - "version": "3.0.3", - "bundled": true, - "optional": true + "version": "3.1.1", + "bundled": true } } }, @@ -2655,6 +3173,26 @@ "rimraf": "2" } }, + "functional-red-black-tree": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", + "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=" + }, + "gauge": { + "version": "2.7.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", + "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", + "requires": { + "aproba": "^1.0.3", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.0", + "object-assign": "^4.1.0", + "signal-exit": "^3.0.0", + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1", + "wide-align": "^1.1.0" + } + }, "gaxios": { "version": "1.8.4", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.8.4.tgz", @@ -2785,6 +3323,29 @@ "ini": "^1.3.4" } }, + "global-modules": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/global-modules/-/global-modules-2.0.0.tgz", + "integrity": "sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A==", + "requires": { + "global-prefix": "^3.0.0" + } + }, + "global-prefix": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-prefix/-/global-prefix-3.0.0.tgz", + "integrity": "sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg==", + "requires": { + "ini": "^1.3.5", + "kind-of": "^6.0.2", + "which": "^1.3.1" + } + }, + "globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==" + }, "google-auth-library": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-3.1.2.tgz", @@ -2820,493 +3381,87 @@ "retry-request": "^4.0.0", "semver": "^6.0.0", "walkdir": "^0.3.2" - }, - "dependencies": { - "semver": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.2.0.tgz", - "integrity": "sha512-jdFC1VdUGT/2Scgbimf7FSx9iJLXoqfglSF+gJeuNWVpiE37OIbc1jywR/GJyFdz3mnkz2/id0L0J/cr0izR5A==" - } - } - }, - "google-p12-pem": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.4.tgz", - "integrity": "sha512-SwLAUJqUfTB2iS+wFfSS/G9p7bt4eWcc2LyfvmUXe7cWp6p3mpxDo6LLI29MXdU6wvPcQ/up298X7GMC5ylAlA==", - "requires": { - "node-forge": "^0.8.0", - "pify": "^4.0.0" - } - }, - "google-proto-files": { - "version": "0.20.0", - "resolved": "https://registry.npmjs.org/google-proto-files/-/google-proto-files-0.20.0.tgz", - "integrity": "sha512-ORU+XhOeDv/UPtnCYLkO1ItmfhRCRPR3ZoeVQ7GfVzEs7PVitPIhsYlY5ZzG8XXnsdmtK27ENurfQ1jhAWpZHg==", - "requires": { - "@google-cloud/promisify": "^0.4.0", - "protobufjs": "^6.8.0", - "walkdir": "^0.3.0" - } - }, - "got": { - "version": "6.7.1", - "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz", - "integrity": "sha1-JAzQV4WpoY5WHcG0S0HHY+8ejbA=", - "requires": { - "create-error-class": "^3.0.0", - "duplexer3": "^0.1.4", - "get-stream": "^3.0.0", - "is-redirect": "^1.0.0", - "is-retry-allowed": "^1.0.0", - "is-stream": "^1.0.0", - "lowercase-keys": "^1.0.0", - "safe-buffer": "^5.0.1", - "timed-out": "^4.0.0", - "unzip-response": "^2.0.1", - "url-parse-lax": "^1.0.0" - } - }, - "graceful-fs": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.0.tgz", - "integrity": "sha512-jpSvDPV4Cq/bgtpndIWbI5hmYxhQGHPC4d4cqBPb4DLniCfhJokdXhwhaDuLBGLQdvvRum/UiX6ECVIPvDXqdg==" - }, - "growl": { - "version": "1.10.5", - "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", - "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", - "dev": true - }, - "grpc": { - "version": "1.22.2", - "resolved": "https://registry.npmjs.org/grpc/-/grpc-1.22.2.tgz", - "integrity": "sha512-gaK59oAA5/mlOIn+hQO5JROPoAzsaGRpEMcrAayW5WGETS8QScpBoQ+XBxEWAAF0kbeGIELuGRCVEObKS1SLmw==", - "requires": { - "lodash.camelcase": "^4.3.0", - "lodash.clone": "^4.5.0", - "nan": "^2.13.2", - "node-pre-gyp": "^0.13.0", - "protobufjs": "^5.0.3" - }, - "dependencies": { - "abbrev": { - "version": "1.1.1", - "bundled": true - }, - "ansi-regex": { - "version": "2.1.1", - "bundled": true - }, - "aproba": { - "version": "1.2.0", - "bundled": true - }, - "are-we-there-yet": { - "version": "1.1.5", - "bundled": true, - "requires": { - "delegates": "^1.0.0", - "readable-stream": "^2.0.6" - } - }, - "balanced-match": { - "version": "1.0.0", - "bundled": true - }, - "brace-expansion": { - "version": "1.1.11", - "bundled": true, - "requires": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "chownr": { - "version": "1.1.1", - "bundled": true - }, - "code-point-at": { - "version": "1.1.0", - "bundled": true - }, - "concat-map": { - "version": "0.0.1", - "bundled": true - }, - "console-control-strings": { - "version": "1.1.0", - "bundled": true - }, - "core-util-is": { - "version": "1.0.2", - "bundled": true - }, - "deep-extend": { - "version": "0.6.0", - "bundled": true - }, - "delegates": { - "version": "1.0.0", - "bundled": true - }, - "detect-libc": { - "version": "1.0.3", - "bundled": true - }, - "fs-minipass": { - "version": "1.2.6", - "bundled": true, - "requires": { - "minipass": "^2.2.1" - } - }, - "fs.realpath": { - "version": "1.0.0", - "bundled": true - }, - "gauge": { - "version": "2.7.4", - "bundled": true, - "requires": { - "aproba": "^1.0.3", - "console-control-strings": "^1.0.0", - "has-unicode": "^2.0.0", - "object-assign": "^4.1.0", - "signal-exit": "^3.0.0", - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1", - "wide-align": "^1.1.0" - } - }, - "has-unicode": { - "version": "2.0.1", - "bundled": true - }, - "iconv-lite": { - "version": "0.4.23", - "bundled": true, - "requires": { - "safer-buffer": ">= 2.1.2 < 3" - } - }, - "ignore-walk": { - "version": "3.0.1", - "bundled": true, - "requires": { - "minimatch": "^3.0.4" - } - }, - "inflight": { - "version": "1.0.6", - "bundled": true, - "requires": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "inherits": { - "version": "2.0.3", - "bundled": true - }, - "ini": { - "version": "1.3.5", - "bundled": true - }, - "is-fullwidth-code-point": { - "version": "1.0.0", - "bundled": true, - "requires": { - "number-is-nan": "^1.0.0" - } - }, - "isarray": { - "version": "1.0.0", - "bundled": true - }, - "minimatch": { - "version": "3.0.4", - "bundled": true, - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "minimist": { - "version": "1.2.0", - "bundled": true - }, - "minipass": { - "version": "2.3.5", - "bundled": true, - "requires": { - "safe-buffer": "^5.1.2", - "yallist": "^3.0.0" - } - }, - "minizlib": { - "version": "1.2.1", - "bundled": true, - "requires": { - "minipass": "^2.2.1" - } - }, - "mkdirp": { - "version": "0.5.1", - "bundled": true, - "requires": { - "minimist": "0.0.8" - }, - "dependencies": { - "minimist": { - "version": "0.0.8", - "bundled": true - } - } - }, - "needle": { - "version": "2.4.0", - "bundled": true, - "requires": { - "debug": "^3.2.6", - "iconv-lite": "^0.4.4", - "sax": "^1.2.4" - }, - "dependencies": { - "debug": { - "version": "3.2.6", - "bundled": true, - "requires": { - "ms": "^2.1.1" - } - }, - "ms": { - "version": "2.1.2", - "bundled": true - } - } - }, - "node-pre-gyp": { - "version": "0.13.0", - "bundled": true, - "requires": { - "detect-libc": "^1.0.2", - "mkdirp": "^0.5.1", - "needle": "^2.2.1", - "nopt": "^4.0.1", - "npm-packlist": "^1.1.6", - "npmlog": "^4.0.2", - "rc": "^1.2.7", - "rimraf": "^2.6.1", - "semver": "^5.3.0", - "tar": "^4" - } - }, - "nopt": { - "version": "4.0.1", - "bundled": true, - "requires": { - "abbrev": "1", - "osenv": "^0.1.4" - } - }, - "npm-bundled": { - "version": "1.0.6", - "bundled": true - }, - "npm-packlist": { - "version": "1.4.1", - "bundled": true, - "requires": { - "ignore-walk": "^3.0.1", - "npm-bundled": "^1.0.1" - } - }, - "npmlog": { - "version": "4.1.2", - "bundled": true, - "requires": { - "are-we-there-yet": "~1.1.2", - "console-control-strings": "~1.1.0", - "gauge": "~2.7.3", - "set-blocking": "~2.0.0" - } - }, - "number-is-nan": { - "version": "1.0.1", - "bundled": true - }, - "object-assign": { - "version": "4.1.1", - "bundled": true - }, - "once": { - "version": "1.4.0", - "bundled": true, - "requires": { - "wrappy": "1" - } - }, - "os-homedir": { - "version": "1.0.2", - "bundled": true - }, - "os-tmpdir": { - "version": "1.0.2", - "bundled": true - }, - "osenv": { - "version": "0.1.5", - "bundled": true, - "requires": { - "os-homedir": "^1.0.0", - "os-tmpdir": "^1.0.0" - } - }, - "path-is-absolute": { - "version": "1.0.1", - "bundled": true - }, - "process-nextick-args": { - "version": "2.0.1", - "bundled": true - }, - "protobufjs": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-5.0.3.tgz", - "integrity": "sha512-55Kcx1MhPZX0zTbVosMQEO5R6/rikNXd9b6RQK4KSPcrSIIwoXTtebIczUrXlwaSrbz4x8XUVThGPob1n8I4QA==", - "requires": { - "ascli": "~1", - "bytebuffer": "~5", - "glob": "^7.0.5", - "yargs": "^3.10.0" - } - }, - "rc": { - "version": "1.2.8", - "bundled": true, - "requires": { - "deep-extend": "^0.6.0", - "ini": "~1.3.0", - "minimist": "^1.2.0", - "strip-json-comments": "~2.0.1" - } - }, - "readable-stream": { - "version": "2.3.6", - "bundled": true, - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "rimraf": { - "version": "2.6.3", - "bundled": true, - "requires": { - "glob": "^7.1.3" - }, - "dependencies": { - "glob": { - "version": "7.1.4", - "bundled": true, - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - } - } - }, - "safe-buffer": { - "version": "5.1.2", - "bundled": true - }, - "safer-buffer": { - "version": "2.1.2", - "bundled": true - }, - "sax": { - "version": "1.2.4", - "bundled": true - }, - "semver": { - "version": "5.7.0", - "bundled": true - }, - "set-blocking": { - "version": "2.0.0", - "bundled": true - }, - "signal-exit": { - "version": "3.0.1", - "bundled": true - }, - "string-width": { - "version": "1.0.2", - "bundled": true, - "requires": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" - } - }, - "string_decoder": { - "version": "1.1.1", - "bundled": true, - "requires": { - "safe-buffer": "~5.1.0" - } - }, - "strip-ansi": { - "version": "3.0.1", - "bundled": true, - "requires": { - "ansi-regex": "^2.0.0" - } - }, - "strip-json-comments": { - "version": "2.0.1", - "bundled": true - }, - "tar": { - "version": "4.4.10", - "bundled": true, - "requires": { - "chownr": "^1.1.1", - "fs-minipass": "^1.2.5", - "minipass": "^2.3.5", - "minizlib": "^1.2.1", - "mkdirp": "^0.5.0", - "safe-buffer": "^5.1.2", - "yallist": "^3.0.3" - } - }, - "util-deprecate": { - "version": "1.0.2", - "bundled": true - }, - "wide-align": { - "version": "1.1.3", - "bundled": true, + }, + "dependencies": { + "semver": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.2.0.tgz", + "integrity": "sha512-jdFC1VdUGT/2Scgbimf7FSx9iJLXoqfglSF+gJeuNWVpiE37OIbc1jywR/GJyFdz3mnkz2/id0L0J/cr0izR5A==" + } + } + }, + "google-p12-pem": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.4.tgz", + "integrity": "sha512-SwLAUJqUfTB2iS+wFfSS/G9p7bt4eWcc2LyfvmUXe7cWp6p3mpxDo6LLI29MXdU6wvPcQ/up298X7GMC5ylAlA==", + "requires": { + "node-forge": "^0.8.0", + "pify": "^4.0.0" + } + }, + "google-proto-files": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/google-proto-files/-/google-proto-files-0.20.0.tgz", + "integrity": "sha512-ORU+XhOeDv/UPtnCYLkO1ItmfhRCRPR3ZoeVQ7GfVzEs7PVitPIhsYlY5ZzG8XXnsdmtK27ENurfQ1jhAWpZHg==", + "requires": { + "@google-cloud/promisify": "^0.4.0", + "protobufjs": "^6.8.0", + "walkdir": "^0.3.0" + } + }, + "got": { + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz", + "integrity": "sha1-JAzQV4WpoY5WHcG0S0HHY+8ejbA=", + "requires": { + "create-error-class": "^3.0.0", + "duplexer3": "^0.1.4", + "get-stream": "^3.0.0", + "is-redirect": "^1.0.0", + "is-retry-allowed": "^1.0.0", + "is-stream": "^1.0.0", + "lowercase-keys": "^1.0.0", + "safe-buffer": "^5.0.1", + "timed-out": "^4.0.0", + "unzip-response": "^2.0.1", + "url-parse-lax": "^1.0.0" + } + }, + "graceful-fs": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.0.tgz", + "integrity": "sha512-jpSvDPV4Cq/bgtpndIWbI5hmYxhQGHPC4d4cqBPb4DLniCfhJokdXhwhaDuLBGLQdvvRum/UiX6ECVIPvDXqdg==" + }, + "growl": { + "version": "1.10.5", + "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", + "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", + "dev": true + }, + "grpc": { + "version": "1.24.3", + "resolved": "https://registry.npmjs.org/grpc/-/grpc-1.24.3.tgz", + "integrity": "sha512-EDemzuZTfhM0hgrXqC4PtR76O3t+hTIYJYR5vgiW0yt2WJqo4mhxUqZUirzUQz34Psz7dbLp38C6Cl7Ij2vXRQ==", + "requires": { + "@types/bytebuffer": "^5.0.40", + "lodash.camelcase": "^4.3.0", + "lodash.clone": "^4.5.0", + "nan": "^2.13.2", + "node-pre-gyp": "^0.15.0", + "protobufjs": "^5.0.3" + }, + "dependencies": { + "protobufjs": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-5.0.3.tgz", + "integrity": "sha512-55Kcx1MhPZX0zTbVosMQEO5R6/rikNXd9b6RQK4KSPcrSIIwoXTtebIczUrXlwaSrbz4x8XUVThGPob1n8I4QA==", "requires": { - "string-width": "^1.0.2 || 2" + "ascli": "~1", + "bytebuffer": "~5", + "glob": "^7.0.5", + "yargs": "^3.10.0" } }, - "wrappy": { - "version": "1.0.2", - "bundled": true - }, - "yallist": { - "version": "3.0.3", - "bundled": true - }, "yargs": { "version": "3.32.0", "resolved": "https://registry.npmjs.org/yargs/-/yargs-3.32.0.tgz", @@ -3351,6 +3506,11 @@ } } }, + "hang": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/hang/-/hang-1.0.0.tgz", + "integrity": "sha1-ZwUIeYRENeAq4ECcT0VTxkOOHXE=" + }, "har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", @@ -3390,6 +3550,11 @@ "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" }, + "has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=" + }, "has-value": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz", @@ -3443,6 +3608,14 @@ } } }, + "hashish": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/hashish/-/hashish-0.0.4.tgz", + "integrity": "sha1-bWC8b/r3Ebav1g5CbQd5iAFOZVQ=", + "requires": { + "traverse": ">=0.2.4" + } + }, "he": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", @@ -3454,6 +3627,19 @@ "resolved": "https://registry.npmjs.org/headers/-/headers-0.9.6.tgz", "integrity": "sha1-xNtiA0bSOlgGg4hmDnP4+gac3x4=" }, + "hh-mm-ss": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/hh-mm-ss/-/hh-mm-ss-1.2.0.tgz", + "integrity": "sha512-f4I9Hz1dLpX/3mrEs7yq30+FiuO3tt5NWAqAGeBTaoeoBfB8vhcQ3BphuDc5DjZb/K809agqrAaFlP0jhEU/8w==", + "requires": { + "zero-fill": "^2.2.3" + } + }, + "html-entities": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-1.3.1.tgz", + "integrity": "sha512-rhE/4Z3hIhzHAUKbW8jVcCyuT5oJCXXqhN/6mXXVCpzTmvJnoH2HL/bt3EZ6p55jbFJBeAe1ZNpL5BugLujxNA==" + }, "http": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/http/-/http-0.0.0.tgz", @@ -3480,6 +3666,11 @@ "sshpk": "^1.7.0" } }, + "https": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/https/-/https-1.0.0.tgz", + "integrity": "sha1-PDfHrhqO65ZpBKKtHpdaGUt+06Q=" + }, "https-proxy-agent": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.4.tgz", @@ -3504,6 +3695,11 @@ } } }, + "human-signals": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-1.1.1.tgz", + "integrity": "sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw==" + }, "iconv-lite": { "version": "0.4.23", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz", @@ -3517,16 +3713,38 @@ "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" }, + "ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==" + }, "ignore-by-default": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", "integrity": "sha1-SMptcvbGo68Aqa1K5odr44ieKwk=" }, + "ignore-walk": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz", + "integrity": "sha512-m7o6xuOaT1aqheYHKf8W6J5pYH85ZI9w077erOzLje3JsB1gkafkAhHHY19dqjulgIZHFm32Cp5uNZgcQqdJKw==", + "requires": { + "minimatch": "^3.0.4" + } + }, "immediate": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=" }, + "import-fresh": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", + "integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==", + "requires": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + } + }, "import-lazy": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/import-lazy/-/import-lazy-2.1.0.tgz", @@ -3561,6 +3779,72 @@ "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" }, + "inquirer": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.5.2.tgz", + "integrity": "sha512-cntlB5ghuB0iuO65Ovoi8ogLHiWGs/5yNrtUcKjFhSSiVeAIVpD7koaSU9RM8mpXw5YDi9RdYXGQMaOURB7ycQ==", + "requires": { + "ansi-escapes": "^3.2.0", + "chalk": "^2.4.2", + "cli-cursor": "^2.1.0", + "cli-width": "^2.0.0", + "external-editor": "^3.0.3", + "figures": "^2.0.0", + "lodash": "^4.17.12", + "mute-stream": "0.0.7", + "run-async": "^2.2.0", + "rxjs": "^6.4.0", + "string-width": "^2.1.0", + "strip-ansi": "^5.1.0", + "through": "^2.3.6" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=" + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + }, + "dependencies": { + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "requires": { + "ansi-regex": "^3.0.0" + } + } + } + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "requires": { + "ansi-regex": "^4.1.0" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==" + } + } + } + } + }, "invert-kv": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz", @@ -3603,9 +3887,9 @@ } }, "is-buffer": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.3.tgz", - "integrity": "sha512-U15Q7MXTuZlrbymiz95PJpZxu8IlipAp4dtS3wOdgPXx3mqBnslrWU14kxfHB+Py/+2PVKSr37dMAgM2A4uArw==" + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.4.tgz", + "integrity": "sha512-Kq1rokWXOPXWuaMAqZiJW4XxsmD9zGx9q4aePabbn3qCRGedtH7Cm+zV8WETitMfu1wdh+Rvd6w5egwSngUX2A==" }, "is-ci": { "version": "1.2.1", @@ -3739,6 +4023,11 @@ "isobject": "^3.0.1" } }, + "is-promise": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", + "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=" + }, "is-property": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", @@ -3794,6 +4083,20 @@ "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, + "js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" + }, + "js-yaml": { + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", + "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", @@ -3817,6 +4120,11 @@ "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" }, + "json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=" + }, "json-stringify-safe": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", @@ -3887,9 +4195,9 @@ } }, "kind-of": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", - "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" }, "latest-version": { "version": "3.1.0", @@ -3915,6 +4223,15 @@ "invert-kv": "^1.0.0" } }, + "levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", + "requires": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + } + }, "lie": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", @@ -3928,6 +4245,17 @@ "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz", "integrity": "sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc=" }, + "loads": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/loads/-/loads-0.0.4.tgz", + "integrity": "sha1-l/MBY5fnDd/0gLgexjjO6iKgqZw=", + "requires": { + "failure": "1.1.x", + "one-time": "0.0.x", + "xhr-response": "1.0.x", + "xhr-status": "1.0.x" + } + }, "locate-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", @@ -3938,9 +4266,9 @@ } }, "lodash": { - "version": "4.17.14", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.14.tgz", - "integrity": "sha512-mmKYbW3GLuJeX+iGP+Y7Gp1AiGHGbXHCOh/jZmrawMmsE7MS4znI3RL2FsjbqOyMayHInjOeykW7PEajUk1/xw==" + "version": "4.17.20", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", + "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" }, "lodash.at": { "version": "4.6.0", @@ -4015,6 +4343,15 @@ "yallist": "^3.0.2" } }, + "m3u8stream": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/m3u8stream/-/m3u8stream-0.7.1.tgz", + "integrity": "sha512-z6ldnAdhbuWOL6LmMkwptSZGzj+qbRytMKLTbNicwF/bJMjf9U9lqD57RNQUFecvWadEkzy6PDjcNJFFgi19uQ==", + "requires": { + "miniget": "^1.6.1", + "sax": "^1.2.4" + } + }, "make-dir": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", @@ -4056,6 +4393,21 @@ "object-visit": "^1.0.0" } }, + "media-split": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/media-split/-/media-split-5.1.0.tgz", + "integrity": "sha512-Em/qWDcJmL2drPYLrTCEK2SPjvVpRALlLKblQKp5W191Km/MMonJOYYIjV4FUdW1pGxeFIlJSHRthwJFbGxQHg==", + "requires": { + "chalk": "^2.4.2", + "cli-progress": "^2.1.1", + "eslint": "^5.16.0", + "global-modules": "^2.0.0", + "miniget": "^1.5.1", + "sanitize-filename": "^1.6.1", + "yargs": "^13.2.4", + "ytdl-core": "^0.29.2" + } + }, "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -4076,6 +4428,11 @@ "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" }, + "merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==" + }, "methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", @@ -4124,6 +4481,11 @@ "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==" }, + "miniget": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/miniget/-/miniget-1.7.0.tgz", + "integrity": "sha512-yrgaDSMRzrfYTkudB4Y6xK8pCb7oAH2bvfv6iPY2m6CedZfs9yK4b/ofh0Vzv08hCYXH/HHkoS8an6fkWtOAQA==" + }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -4133,9 +4495,26 @@ } }, "minimist": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + }, + "minipass": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz", + "integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==", + "requires": { + "safe-buffer": "^5.1.2", + "yallist": "^3.0.0" + } + }, + "minizlib": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz", + "integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==", + "requires": { + "minipass": "^2.9.0" + } }, "mixin-deep": { "version": "1.3.2", @@ -4157,11 +4536,11 @@ } }, "mkdirp": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", "requires": { - "minimist": "0.0.8" + "minimist": "^1.2.5" } }, "mocha": { @@ -4206,6 +4585,21 @@ "path-is-absolute": "^1.0.0" } }, + "minimist": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", + "dev": true + }, + "mkdirp": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "dev": true, + "requires": { + "minimist": "0.0.8" + } + }, "supports-color": { "version": "5.4.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz", @@ -4262,6 +4656,11 @@ } } }, + "mute-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", + "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=" + }, "nan": { "version": "2.14.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz", @@ -4285,6 +4684,11 @@ "to-regex": "^3.0.1" } }, + "natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=" + }, "ndjson": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/ndjson/-/ndjson-1.5.0.tgz", @@ -4297,9 +4701,9 @@ }, "dependencies": { "minimist": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", - "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" }, "through2": { "version": "2.0.5", @@ -4312,6 +4716,31 @@ } } }, + "needle": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/needle/-/needle-2.5.2.tgz", + "integrity": "sha512-LbRIwS9BfkPvNwNHlsA41Q29kL2L/6VaOJ0qisM5lLWsTV3nP15abO5ITL6L81zqFhzjRKDAYjpcBcwM0AVvLQ==", + "requires": { + "debug": "^3.2.6", + "iconv-lite": "^0.4.4", + "sax": "^1.2.4" + }, + "dependencies": { + "debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "negotiator": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", @@ -4323,15 +4752,48 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==" }, "node-fetch": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", - "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" }, "node-forge": { "version": "0.8.5", "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.8.5.tgz", "integrity": "sha512-vFMQIWt+J/7FLNyKouZ9TazT74PRV3wgv9UT4cRjC8BffxFbKXkgIWR42URCPSnHm/QDz6BOlb2Q0U4+VQT67Q==" }, + "node-http-xhr": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/node-http-xhr/-/node-http-xhr-1.3.4.tgz", + "integrity": "sha512-0bA08/2RKWxw6pMkOVd3KP+0F5+ifQLMMTDxrCgxlgkoU1N8DhCbCSAYEqpgaVYM2smvbVVewiXjW+8AyoLfxQ==" + }, + "node-pre-gyp": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.15.0.tgz", + "integrity": "sha512-7QcZa8/fpaU/BKenjcaeFF9hLz2+7S9AqyXFhlH/rilsQ/hPZKK32RtR5EQHJElgu+q5RfbJ34KriI79UWaorA==", + "requires": { + "detect-libc": "^1.0.2", + "mkdirp": "^0.5.3", + "needle": "^2.5.0", + "nopt": "^4.0.1", + "npm-packlist": "^1.1.6", + "npmlog": "^4.0.2", + "rc": "^1.2.7", + "rimraf": "^2.6.1", + "semver": "^5.3.0", + "tar": "^4.4.2" + }, + "dependencies": { + "nopt": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz", + "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==", + "requires": { + "abbrev": "1", + "osenv": "^0.1.4" + } + } + } + }, "nodemon": { "version": "1.19.1", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-1.19.1.tgz", @@ -4377,6 +4839,29 @@ "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" }, + "npm-bundled": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.1.tgz", + "integrity": "sha512-gqkfgGePhTpAEgUsGEgcq1rqPXA+tv/aVBlgEzfXwA1yiUJF7xtEt3CtVwOjNYQOVknDk0F20w58Fnm3EtG0fA==", + "requires": { + "npm-normalize-package-bin": "^1.0.1" + } + }, + "npm-normalize-package-bin": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-1.0.1.tgz", + "integrity": "sha512-EPfafl6JL5/rU+ot6P3gRSCpPDW5VmIzX959Ob1+ySFUuuYHWHekXpwdUZcKP5C+DS4GEtdJluwBjnsNDl+fSA==" + }, + "npm-packlist": { + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.8.tgz", + "integrity": "sha512-5+AZgwru5IevF5ZdnFglB5wNlHG1AOOuw28WhUq8/8emhBmLv6jX5by4WJCh7lW0uSYZYS6DXqIsyZVIXRZU9A==", + "requires": { + "ignore-walk": "^3.0.1", + "npm-bundled": "^1.0.1", + "npm-normalize-package-bin": "^1.0.1" + } + }, "npm-run-path": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", @@ -4385,6 +4870,17 @@ "path-key": "^2.0.0" } }, + "npmlog": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", + "requires": { + "are-we-there-yet": "~1.1.2", + "console-control-strings": "~1.1.0", + "gauge": "~2.7.3", + "set-blocking": "~2.0.0" + } + }, "number-is-nan": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", @@ -4475,6 +4971,11 @@ "wrappy": "1" } }, + "one-time": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/one-time/-/one-time-0.0.4.tgz", + "integrity": "sha1-+M33eISCb+Tf+T46nMN7HkSAdC4=" + }, "onetime": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.0.tgz", @@ -4483,17 +4984,57 @@ "mimic-fn": "^2.1.0" } }, + "optimist": { + "version": "0.3.7", + "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.3.7.tgz", + "integrity": "sha1-yQlBrVnkJzMokjB00s8ufLxuwNk=", + "requires": { + "wordwrap": "~0.0.2" + } + }, + "optionator": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", + "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", + "requires": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.6", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "word-wrap": "~1.2.3" + } + }, "optjs": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/optjs/-/optjs-3.2.2.tgz", "integrity": "sha1-aabOicRCpEQDFBrS+bNwvVu29O4=" }, + "os-homedir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=" + }, "os-locale": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", "integrity": "sha1-IPnxeuKe00XoveWDsT0gCYA8FNk=", "requires": { - "lcid": "^1.0.0" + "lcid": "^1.0.0" + } + }, + "os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=" + }, + "osenv": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", + "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", + "requires": { + "os-homedir": "^1.0.0", + "os-tmpdir": "^1.0.0" } }, "p-defer": { @@ -4553,6 +5094,29 @@ "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.10.tgz", "integrity": "sha512-0DTvPVU3ed8+HNXOu5Bs+o//Mbdj9VNQMUOe9oKCwh8l0GNwpTDMKCWbRjgtD291AWnkAgkqA/LOnQS8AmS1tw==" }, + "parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "requires": { + "callsites": "^3.0.0" + } + }, + "parse-time": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/parse-time/-/parse-time-0.1.4.tgz", + "integrity": "sha1-idCrQNPz6Rghvzxr3WHQGqEozxY=", + "requires": { + "object-assign": "^2.0.0" + }, + "dependencies": { + "object-assign": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-2.1.1.tgz", + "integrity": "sha1-Q8NuXVaf+OSBbE76i+AtJpZ8GKo=" + } + } + }, "parseqs": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/parseqs/-/parseqs-0.0.5.tgz", @@ -4751,6 +5315,11 @@ "xtend": "^4.0.0" } }, + "prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=" + }, "prepend-http": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz", @@ -4761,6 +5330,11 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, + "progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==" + }, "protobufjs": { "version": "6.8.8", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz", @@ -4837,9 +5411,9 @@ "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" }, "qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + "version": "6.9.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.3.tgz", + "integrity": "sha512-EbZYNarm6138UKKq46tdx08Yo/q9ZhFoAXAI1meAFd2GtbRDhbZY2WQSICskT0c5q99aFzLG1D4nvTk9tqfXIw==" }, "range-parser": { "version": "1.2.1", @@ -4847,14 +5421,46 @@ "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" }, "raw-body": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.3.tgz", - "integrity": "sha512-9esiElv1BrZoI3rCDuOuKCBRbuApGGaDPQfjSflGxdy4oyzqghxu6klEkkVIvBje+FF0BX9coEv8KqW6X/7njw==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", + "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==", "requires": { - "bytes": "3.0.0", - "http-errors": "1.6.3", - "iconv-lite": "0.4.23", + "bytes": "3.1.0", + "http-errors": "1.7.2", + "iconv-lite": "0.4.24", "unpipe": "1.0.0" + }, + "dependencies": { + "http-errors": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", + "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.1", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.0" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "setprototypeof": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", + "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" + }, + "statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" + } } }, "rc": { @@ -4869,9 +5475,9 @@ }, "dependencies": { "minimist": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", - "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" } } }, @@ -4908,6 +5514,11 @@ "safe-regex": "^1.1.0" } }, + "regexpp": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-2.0.1.tgz", + "integrity": "sha512-lv0M6+TkDVniA3aD1Eg0DVpfU/booSu7Eev3TDO/mZKHBfVjgCGTV4t4buppESEYDtkArYFOxTJWv6S5C+iaNw==" + }, "registry-auth-token": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.4.0.tgz", @@ -4941,9 +5552,9 @@ "integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc=" }, "request": { - "version": "2.88.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", - "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", + "version": "2.88.2", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", + "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", "requires": { "aws-sign2": "~0.7.0", "aws4": "^1.8.0", @@ -4952,7 +5563,7 @@ "extend": "~3.0.2", "forever-agent": "~0.6.1", "form-data": "~2.3.2", - "har-validator": "~5.1.0", + "har-validator": "~5.1.3", "http-signature": "~1.2.0", "is-typedarray": "~1.0.0", "isstream": "~0.1.2", @@ -4962,7 +5573,7 @@ "performance-now": "^2.1.0", "qs": "~6.5.2", "safe-buffer": "^5.1.2", - "tough-cookie": "~2.4.3", + "tough-cookie": "~2.5.0", "tunnel-agent": "^0.6.0", "uuid": "^3.3.2" }, @@ -4976,6 +5587,20 @@ "combined-stream": "^1.0.6", "mime-types": "^2.1.12" } + }, + "qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + }, + "tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "requires": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + } } } }, @@ -4998,6 +5623,29 @@ "lodash": "^4.17.11" } }, + "requests": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/requests/-/requests-0.3.0.tgz", + "integrity": "sha512-1B6nkiHjC1O1cSgFhEwkc+xd8vuj04h7xSmCg5yI8nmhCIKbPkX47od8erQ2pokBt5qxUO7dwP4jplXD6k6ISA==", + "requires": { + "axo": "0.0.x", + "eventemitter3": "~4.0.0", + "extendible": "0.1.x", + "hang": "1.0.x", + "loads": "0.0.x", + "node-http-xhr": "~1.3.0", + "xhr-send": "1.0.x" + } + }, + "require": { + "version": "2.4.20", + "resolved": "https://registry.npmjs.org/require/-/require-2.4.20.tgz", + "integrity": "sha1-Zstrqqu2XeinHXk/XGX9GE83mLY=", + "requires": { + "std": "0.1.40", + "uglify-js": "2.3.0" + } + }, "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -5008,11 +5656,40 @@ "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" }, + "resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==" + }, "resolve-url": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz", "integrity": "sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=" }, + "restore-cursor": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", + "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=", + "requires": { + "onetime": "^2.0.0", + "signal-exit": "^3.0.2" + }, + "dependencies": { + "mimic-fn": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", + "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==" + }, + "onetime": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", + "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=", + "requires": { + "mimic-fn": "^1.0.0" + } + } + } + }, "ret": { "version": "0.1.15", "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", @@ -5050,6 +5727,22 @@ "glob": "^7.1.3" } }, + "run-async": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.0.tgz", + "integrity": "sha512-xJTbh/d7Lm7SBhc1tNvTpeCHaEzoyxPrqNlvSdMfBTYwaY++UJFyXUOxAtsRUXjlqOfj8luNaR9vjCh4KeV+pg==", + "requires": { + "is-promise": "^2.1.0" + } + }, + "rxjs": { + "version": "6.5.5", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.5.tgz", + "integrity": "sha512-WfQI+1gohdf0Dai/Bbmk5L5ItH5tYqm3ki2c5GdWhKjalzjg93N3avFjVStyZZz+A2Em+ZxKH5bNghw9UeylGQ==", + "requires": { + "tslib": "^1.9.0" + } + }, "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", @@ -5068,6 +5761,14 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "sanitize-filename": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/sanitize-filename/-/sanitize-filename-1.6.3.tgz", + "integrity": "sha512-y/52Mcy7aw3gRm7IrcGDFx/bCk4AhRh2eI9luHOQM86nZsqwiRkkq2GekHXBBD+SmPidc8i2PqtYZl+pWJ8Oeg==", + "requires": { + "truncate-utf8-bytes": "^1.0.0" + } + }, "sax": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", @@ -5176,6 +5877,23 @@ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" }, + "slice-ansi": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz", + "integrity": "sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ==", + "requires": { + "ansi-styles": "^3.2.0", + "astral-regex": "^1.0.0", + "is-fullwidth-code-point": "^2.0.0" + }, + "dependencies": { + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" + } + } + }, "snakeize": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/snakeize/-/snakeize-0.1.0.tgz", @@ -5428,11 +6146,11 @@ "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=" }, "source-map-resolve": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.2.tgz", - "integrity": "sha512-MjqsvNwyz1s0k81Goz/9vRBe9SZdB09Bdw+/zYyO+3CuPk6fouTaxscHkgtE8jKvf01kVfl8riHzERQ/kefaSA==", + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.3.tgz", + "integrity": "sha512-Htz+RnsXWk5+P2slx5Jh3Q66vhQj1Cllm0zvnaY98+NFx+Dv2CF/f5O/t8x+KaNdrdIAsruNzoh/KpialbqAnw==", "requires": { - "atob": "^2.1.1", + "atob": "^2.1.2", "decode-uri-component": "^0.2.0", "resolve-url": "^0.2.1", "source-map-url": "^0.4.0", @@ -5492,6 +6210,11 @@ } } }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" + }, "sshpk": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", @@ -5532,6 +6255,11 @@ "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" }, + "std": { + "version": "0.1.40", + "resolved": "https://registry.npmjs.org/std/-/std-0.1.40.tgz", + "integrity": "sha1-Nnil9lCU2eG2teJu2/wCErg0K3E=" + }, "stealthy-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", @@ -5550,6 +6278,14 @@ "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz", "integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=" }, + "streamify": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/streamify/-/streamify-0.2.9.tgz", + "integrity": "sha512-8pUxeLEef9UO1FxtTt5iikAiyzGI4SZRnGuJ3sz8axZ5Xk+/7ezEV5kuJQsMEFxw7AKYw3xp0Ow+20mmSaJbQQ==", + "requires": { + "hashish": "~0.0.4" + } + }, "streamsearch": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", @@ -5586,6 +6322,11 @@ "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=" }, + "strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==" + }, "strip-json-comments": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", @@ -5604,6 +6345,61 @@ "has-flag": "^3.0.0" } }, + "table": { + "version": "5.4.6", + "resolved": "https://registry.npmjs.org/table/-/table-5.4.6.tgz", + "integrity": "sha512-wmEc8m4fjnob4gt5riFRtTu/6+4rSe12TpAELNSqHMfF3IqnA+CH37USM6/YR3qRZv7e56kAEAtd6nKZaxe0Ug==", + "requires": { + "ajv": "^6.10.2", + "lodash": "^4.17.14", + "slice-ansi": "^2.1.0", + "string-width": "^3.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==" + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "requires": { + "ansi-regex": "^4.1.0" + } + } + } + }, + "tar": { + "version": "4.4.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.13.tgz", + "integrity": "sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA==", + "requires": { + "chownr": "^1.1.1", + "fs-minipass": "^1.2.5", + "minipass": "^2.8.6", + "minizlib": "^1.2.1", + "mkdirp": "^0.5.0", + "safe-buffer": "^5.1.2", + "yallist": "^3.0.3" + } + }, "tar-stream": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.0.tgz", @@ -5686,6 +6482,11 @@ } } }, + "text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=" + }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -5760,6 +6561,11 @@ "repeat-string": "^1.6.1" } }, + "toidentifier": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", + "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" + }, "touch": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz", @@ -5789,6 +6595,19 @@ "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=" }, + "truncate-utf8-bytes": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz", + "integrity": "sha1-QFkjkJWS1W94pYGENLC3hInKXys=", + "requires": { + "utf8-byte-length": "^1.0.1" + } + }, + "tslib": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.11.1.tgz", + "integrity": "sha512-aZW88SY8kQbU7gpV19lN24LtXh/yD4ZZg6qieAJDDg+YBsJcSmLGK9QpnUjAKVG/xefmvJGd1WUmfpT/g6AJGA==" + }, "tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", @@ -5802,6 +6621,14 @@ "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" }, + "type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", + "requires": { + "prelude-ls": "~1.1.2" + } + }, "type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", @@ -5822,6 +6649,31 @@ "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" }, + "uglify-js": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-2.3.0.tgz", + "integrity": "sha1-LN7BbTeKiituz7aYl4TPi3rlSR8=", + "requires": { + "async": "~0.2.6", + "optimist": "~0.3.5", + "source-map": "~0.1.7" + }, + "dependencies": { + "async": { + "version": "0.2.10", + "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz", + "integrity": "sha1-trvgsGdLnXGXCMo43owjfLUmw9E=" + }, + "source-map": { + "version": "0.1.43", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.1.43.tgz", + "integrity": "sha1-wkvBRspRfBRx9drL4lcbK3+eM0Y=", + "requires": { + "amdefine": ">=0.0.4" + } + } + } + }, "undefsafe": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.2.tgz", @@ -5924,9 +6776,9 @@ } }, "upath": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/upath/-/upath-1.1.2.tgz", - "integrity": "sha512-kXpym8nmDmlCBr7nKdIx8P2jNBa+pBpIUFRnKJ4dr8htyYGJFokkr2ZvERRtUN+9SY+JqXouNgUPtv6JQva/2Q==" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/upath/-/upath-1.2.0.tgz", + "integrity": "sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg==" }, "update-notifier": { "version": "2.5.0", @@ -5986,6 +6838,11 @@ "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==" }, + "utf8-byte-length": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/utf8-byte-length/-/utf8-byte-length-1.0.4.tgz", + "integrity": "sha1-9F8VDExm7uloGGUFq5P8u4rWv2E=" + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -6021,6 +6878,11 @@ "resolved": "https://registry.npmjs.org/walkdir/-/walkdir-0.3.2.tgz", "integrity": "sha512-0Twghia4Z5wDGDYWURlhZmI47GvERMCsXIu0QZWVVZyW9ZjpbbZvD9Zy9M6cWiQQRRbAcYajIyKNavaZZDt1Uw==" }, + "when": { + "version": "3.7.8", + "resolved": "https://registry.npmjs.org/when/-/when-3.7.8.tgz", + "integrity": "sha1-xxMLan6gRpPoQs3J56Hyqjmjn4I=" + }, "which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", @@ -6034,6 +6896,14 @@ "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" }, + "wide-align": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", + "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", + "requires": { + "string-width": "^1.0.2 || 2" + } + }, "widest-line": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-2.0.1.tgz", @@ -6076,6 +6946,16 @@ "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.1.4.tgz", "integrity": "sha1-+OGqHuWlPsW/FR/6CXQqatdpeHY=" }, + "word-wrap": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", + "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==" + }, + "wordwrap": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", + "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=" + }, "wrap-ansi": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", @@ -6090,6 +6970,14 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, + "write": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/write/-/write-1.0.3.tgz", + "integrity": "sha512-/lg70HAjtkUgWPVZhZcm+T4hkL8Zbtp1nFNOn3lRrxnlv50SRBv7cR7RqR+GMsd3hUXy9hWBo4CHTbFTcOYwig==", + "requires": { + "mkdirp": "^0.5.1" + } + }, "write-file-atomic": { "version": "2.4.3", "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.3.tgz", @@ -6101,18 +6989,30 @@ } }, "ws": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.2.0.tgz", - "integrity": "sha512-+SqNqFbwTm/0DC18KYzIsMTnEWpLwJsiasW/O17la4iDRRIO9uaHbvKiAS3AHgTiuuWerK/brj4O6MYZkei9xg==", - "requires": { - "async-limiter": "^1.0.0" - } + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.2.1.tgz", + "integrity": "sha512-sucePNSafamSKoOqoNfBd8V0StlkzJKL2ZAhGQinCfNQ+oacw+Pk7lcdAElecBF2VkLNZRiIb5Oi1Q5lVUVt2A==" }, "xdg-basedir": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-3.0.0.tgz", "integrity": "sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=" }, + "xhr-response": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/xhr-response/-/xhr-response-1.0.1.tgz", + "integrity": "sha1-r/46CFRLpyGG5NxSzQImZx2Gc/4=" + }, + "xhr-send": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/xhr-send/-/xhr-send-1.0.0.tgz", + "integrity": "sha1-GkDHPl2yAo7gj+kPXD1tx/eWqFQ=" + }, + "xhr-status": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/xhr-status/-/xhr-status-1.0.1.tgz", + "integrity": "sha512-VF0WSqtmkf56OmF26LCWsWvRb1a+WYGdHDoQnPPCVUQTM8CVUAOBcUDsm7nP7SQcgEEdrvF4DmhEADuXdGieyw==" + }, "xmlhttprequest-ssl": { "version": "1.5.5", "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.5.5.tgz", @@ -6273,9 +7173,9 @@ } }, "yargs-parser": { - "version": "13.1.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.1.tgz", - "integrity": "sha512-oVAVsHz6uFrg3XQheFII8ESO2ssAf9luWuAd6Wexsu4F3OtIW0o8IribPXYrD4WC24LWtPrJlGy87y5udK+dxQ==", + "version": "13.1.2", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.2.tgz", + "integrity": "sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==", "requires": { "camelcase": "^5.0.0", "decamelize": "^1.2.0" @@ -6293,6 +7193,150 @@ "resolved": "https://registry.npmjs.org/yeast/-/yeast-0.1.2.tgz", "integrity": "sha1-AI4G2AlDIMNy28L47XagymyKxBk=" }, + "youtube-dl": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/youtube-dl/-/youtube-dl-3.0.2.tgz", + "integrity": "sha512-LFFfpsYbRLpqKsnb4gzbnyN7fm190tJw3gJVSvfoEfnb/xYIPNT6i9G3jdzPDp/U5cwB3OSq63nUa7rUwxXAGA==", + "requires": { + "debug": "~4.1.1", + "execa": "~3.2.0", + "hh-mm-ss": "~1.2.0", + "mkdirp": "~0.5.1", + "request": "~2.88.0", + "streamify": "~0.2.9", + "universalify": "~0.1.2" + }, + "dependencies": { + "cross-spawn": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.1.tgz", + "integrity": "sha512-u7v4o84SwFpD32Z8IIcPZ6z1/ie24O6RU3RbtL5Y316l3KuHVPx9ItBgWQ6VlfAFnRnTtMUrsQ9MUUTuEZjogg==", + "requires": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "execa": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-3.2.0.tgz", + "integrity": "sha512-kJJfVbI/lZE1PZYDI5VPxp8zXPO9rtxOkhpZ0jMKha56AI9y2gGVC6bkukStQf0ka5Rh15BA5m7cCCH4jmHqkw==", + "requires": { + "cross-spawn": "^7.0.0", + "get-stream": "^5.0.0", + "human-signals": "^1.1.1", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.0", + "onetime": "^5.1.0", + "p-finally": "^2.0.0", + "signal-exit": "^3.0.2", + "strip-final-newline": "^2.0.0" + } + }, + "get-stream": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz", + "integrity": "sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==", + "requires": { + "pump": "^3.0.0" + } + }, + "is-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", + "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==" + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "requires": { + "path-key": "^3.0.0" + } + }, + "p-finally": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-2.0.1.tgz", + "integrity": "sha512-vpm09aKwq6H9phqRQzecoDpD8TmVyGw70qmWlyq5onxY7tqyTTFVvxMykxQSQKILBSFlbXpypIw2T1Ml7+DDtw==" + }, + "path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" + }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "requires": { + "isexe": "^2.0.0" + } + } + } + }, + "ytdl-core": { + "version": "0.29.7", + "resolved": "https://registry.npmjs.org/ytdl-core/-/ytdl-core-0.29.7.tgz", + "integrity": "sha512-mqPuZ1UyspT84lyV1FSU1UAoiQ4H5j52ba3sC1LC43OmL8qNaDIRCGffGlpiakQrc69qsplNWsWEZUF+J5bhJA==", + "requires": { + "html-entities": "^1.1.3", + "m3u8stream": "^0.6.2", + "miniget": "^1.5.3", + "sax": "^1.1.3" + }, + "dependencies": { + "m3u8stream": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/m3u8stream/-/m3u8stream-0.6.5.tgz", + "integrity": "sha512-QZCzhcfUliZfsOboi68QkNcMejPKTEhxE+s1TApvHubDeR8ythm4ViWuYFqgUwZeoHe8q0nsPxOvA3lQvdSzyg==", + "requires": { + "miniget": "^1.6.1", + "sax": "^1.2.4" + } + } + } + }, + "zero-fill": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/zero-fill/-/zero-fill-2.2.3.tgz", + "integrity": "sha1-o97wa6XjmuZEhQu0yirUEStIVek=" + }, "zip-stream": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-2.1.2.tgz", diff --git a/package.json b/package.json index 1a06dfc..d038a95 100644 --- a/package.json +++ b/package.json @@ -11,15 +11,18 @@ "@google-cloud/text-to-speech": "^1.4.1", "@google-cloud/video-intelligence": "^1.5.1", "axios": "^0.18.1", - "body-parser": "^1.18.3", + "body-parser": "^1.19.0", + "child_process": "^1.0.2", "circular-json": "^0.5.9", "cookie-parser": "~1.4.3", + "cors": "^2.8.5", "csv-parser": "^2.3.1", "debug": "~2.6.9", "ejs": "~2.5.7", "exceljs": "^3.3.1", "express": "~4.16.0", "fetch": "^1.1.0", + "ffmpeg": "0.0.4", "fluent-ffmpeg": "^2.1.2", "form-data": "^2.5.1", "fs": "0.0.1-security", @@ -28,19 +31,27 @@ "headers": "^0.9.6", "http": "0.0.0", "http-errors": "~1.6.2", + "https": "^1.0.0", "json2csv": "^4.5.4", + "m3u8stream": "^0.7.1", + "media-split": "^5.1.0", "morgan": "~1.9.0", "multer": "^1.4.1", - "node-fetch": "^2.6.0", + "node-fetch": "^2.6.1", "nodemon": "^1.18.10", + "parse-time": "^0.1.4", "pg": "^7.11.0", "pg-essential": "^1.0.2", "pg-promise": "^8.6.4", - "request": "^2.88.0", + "qs": "^6.9.3", + "request": "^2.88.2", "request-promise": "^4.2.4", + "requests": "^0.3.0", + "require": "^2.4.20", "socket.io": "^2.3.0", "uuid": "^3.3.3", - "yargs": "^13.2.0" + "yargs": "^13.2.0", + "youtube-dl": "^3.0.2" }, "devDependencies": { "chai": "^4.2.0", diff --git a/process.json b/process.json new file mode 100644 index 0000000..52f7007 --- /dev/null +++ b/process.json @@ -0,0 +1,10 @@ +{ + "apps": [ + { + "name": "my-app", + "script": "npm", + "args" : "start", + "ignore_watch": ["public/keyframes"] + } + ] +} \ No newline at end of file diff --git a/public/audio/earcons/beep-08b.wav b/public/audio/earcons/beep-08b.wav new file mode 100644 index 0000000..24a13d4 Binary files /dev/null and b/public/audio/earcons/beep-08b.wav differ diff --git a/public/audio/earcons/beep-09.wav b/public/audio/earcons/beep-09.wav new file mode 100644 index 0000000..71c69f3 Binary files /dev/null and b/public/audio/earcons/beep-09.wav differ diff --git a/public/audio/earcons/ting.wav b/public/audio/earcons/ting.wav new file mode 100644 index 0000000..7fbc54c Binary files /dev/null and b/public/audio/earcons/ting.wav differ diff --git a/public/audio/earcons/vote-down.wav b/public/audio/earcons/vote-down.wav new file mode 100644 index 0000000..725a86b Binary files /dev/null and b/public/audio/earcons/vote-down.wav differ diff --git a/public/audio/earcons/vote-down2.wav b/public/audio/earcons/vote-down2.wav new file mode 100644 index 0000000..cf3b578 Binary files /dev/null and b/public/audio/earcons/vote-down2.wav differ diff --git a/public/audio/earcons/vote-up.wav b/public/audio/earcons/vote-up.wav new file mode 100644 index 0000000..9181565 Binary files /dev/null and b/public/audio/earcons/vote-up.wav differ diff --git a/public/javascripts/audioDescription.js b/public/javascripts/audioDescription.js new file mode 100644 index 0000000..4cbb9ed --- /dev/null +++ b/public/javascripts/audioDescription.js @@ -0,0 +1,1086 @@ +let player; +let youtubeId = ""; +let videotime = 0; +let timeupdater = null; +let sceneIdToListPositionMap = new Map(); +let sceneToBlockMap = new Map(); +let userId; +let prevSceneNum = -1; +let timeAtWhichLastPaused = -1; +let previouslyHighlightedContainerId = "#doesnotexist"; +let enableAi = false; +let autoPauseAtEachScene = true; +let timeElapsedOnPage = 0; +let isTimerPaused = false; +let allSceneArr = []; +let allBlockArr = []; +let audioArr = []; +let allDialogArraySTartTimes = []; +let allDialogArrayEndTimes = []; + +let allSentenceStartTimes = []; +let allSentenceEndTimes = []; +let isSceneExtended = []; // Required to check if a scene has to be auto-paused to play the extended audio +let allSentenceAudioType = []; +let allSentenceSceneNum = []; +let scene_audio_extend_set = new Set(); +let sentenceAudioMap = {}; +let allSentenceInlineStartTimeArr = []; +let allSentenceInlineDurArr = []; +let allSentenceInlineArr = []; + +let currAudio = null; +let currAudioTime; +let wasCurrAudioPaused = false; +let STATE = 0; +let YT_STATE = 0; +let STATE_TIME = -1; +let STATE_CHANGE = 0; +let YT_FLAG = false; +let YT_FLAG_Dial = false; +let YT_FLAG_EXT = false; +let first_extend_flag = true; +let desc_flag = false; +let audio_flag = false; + +$(document).ready(function() { + $("#btn-video-breakpoint-toggle").prop("checked", true); + userId = getUrlVars()["u"]; + enableAi = getUrlVars()["ai"] == "yes" ? true : false; + let href = location.href; + const lastPathSegment = new URL(href).pathname.split("/").pop(); + youtubeId = lastPathSegment; + fetchVideoInfo(youtubeId); + + $("#btnScroll").click(function() { + resetVidModifiedTableData(); + }); + $("#btnSubmit").click(async function() { + let shoudShowAlert = true; + updateAll(shoudShowAlert); + }); + $("#btnTemp").click(async function() { + updateMainTables(); + }); + $("#btn-video-breakpoint-toggle").change(function() { + if ($(this).is(":checked")) { + autoPauseAtEachScene = true; + } else { + autoPauseAtEachScene = false; + } + }); + $("#btn-play-pause").click(function() { + onPlayPauseButtonClick(); + }); + $("#btn-timer").click(function() { + isTimerPaused = !isTimerPaused; + if (isTimerPaused) { + $("#btn-timer").attr("src", "/images/play_white.png"); + } else { + $("#btn-timer").attr("src", "/images/pause_white.png"); + } + }); + setInterval(incrementSeconds, 1000); +}); + +function incrementSeconds() { + if (!isTimerPaused) { + timeElapsedOnPage += 1; + $("#txt-timeElaspsed").html(formatTimeForTimer(timeElapsedOnPage)); + } +} + +function fetchVideoInfo(youtubeId) { + console.log(userId); + // let targetUrl = '/fetchVideoData' + '?videoid=' + youtubeId; + let targetUrl = `/fetchVideoData?videoid=${youtubeId}&userId=${userId}`; + $.get(targetUrl, function(response) { + let vidJson = response; + let vidTitle = vidJson.title; + let vidLength = vidJson.duration; + let blocksJsonArr = vidJson.info; + + overrideData = vidJson.overrideData || []; + overrideData = overrideData.map(over => { + let { deletedScene = "[]", ...rest } = over; + return { + deletedScene: JSON.parse(deletedScene), + ...rest + }; + }); + + populateAllSceneArrAndAllBlockArr(blocksJsonArr); + sceneIdToListPositionMap = populateSceneIdToListPosition(blocksJsonArr); + console.log(sceneIdToListPositionMap); + // for (var [key, value] of sceneIdToListPositionMap) { + // let sceneId = key; + // let sceneNum = sceneId.split('_')[2]-1; + // let randomNum = Math.random(); + // let descType = enableAi ? 'ai' : 'non_ai'; + // let src = `../audio/${youtubeId}/${descType}/${userId}/${sceneNum}.mp3` + '?random='+randomNum; + // // console.log(src); + // let audioFile = new Audio(src); + // audioArr.push(audioFile); + // } + + $("#txtVidtitle").html(vidTitle); + + //setting the data attribute for list-container\ + let listContainer = document.querySelector(".list-container"); + if (overrideData.length > 0) { + listContainer.dataset.sceneDetails = JSON.stringify(overrideData); + } else { + let sceneArr = []; + for (let i = 0; i < blocksJsonArr.length; i++) { + let currBlock = blocksJsonArr[i]; + let childSceneArr = currBlock.child_scenes; + sceneArr.push.apply(sceneArr, childSceneArr); + } + listContainer.dataset.sceneDetails = JSON.stringify(sceneArr); + } + let sceneDetails = JSON.parse(listContainer.dataset.sceneDetails); + sceneIdToListPositionMap = populateSceneIdToListPositionMap(sceneDetails); + initializeDescriptionsUI(); + if (response.qnaList) { + renderQnaList(response.qnaList); + } + }); + + let target_url = "/getDialogTimeStamps" + "?videoId=" + youtubeId; + $.get(target_url, function(response) { + // KeyFramesArr = response; + // debugger; + console.log("Dialog timestamps" + response[0]); + for (val of response) { + allDialogArraySTartTimes.push(val["start_time"]); + allDialogArrayEndTimes.push(val["end_time"]); + // debugger; + } + }); + console.log("Times", allDialogArraySTartTimes); + + let target_url_sentence = + "/getSentences" + "?videoId=" + youtubeId + "&userId=" + userId; + $.get(target_url_sentence, function(response) { + // KeyFramesArr = response; + // debugger; + console.log("sentence start times" + response[0]); + allSentenceArr = response; + for (val of response) { + allSentenceStartTimes.push(val["audio_start_time"]); + // allSentenceEndTimes.push(val['audio_end_time']); + allSentenceAudioType.push(val["audio_type"]); + allSentenceSceneNum.push(val["scene_num"]); + + // let sceneId = val['scene_id']; + let sceneNum = val["scene_num"]; + let randomNum = Math.random(); + // let descType = val['has_ai'] ? 'ai' : 'non_ai'; + let descType = "ai"; + // let sentNumArr = val['sentence_id'].split('_'); + // let sentNum = sentNumArr[sentNumArr.length - 1]; + let audioType = val["audio_type"]; + let path = val["audio_path"].split("/"); + let path_ext = path[path.length - 1]; + let src = + `../audio/${youtubeId}/${descType}/${userId}/${sceneNum}/new/${path_ext}` + + "?random=" + + randomNum; + // let src = path ; + console.log(typeof src); + let audioFile = new Audio(src); + // audioArr.push(audioFile); + + if ( + !scene_audio_extend_set.has(val["scene_num"]) && + val["audio_type"] == "extended" + ) { + scene_audio_extend_set.add(val["scene_num"]); + //create audio object + sentenceAudioMap[val["scene_num"]] = [audioFile]; + } else if (val["audio_type"] == "extended") { + // create audio object + sentenceAudioMap[val["scene_num"]].push(audioFile); + } else if (val["audio_type"] == "inline") { + allSentenceInlineStartTimeArr.push(val["audio_start_time"]); + allSentenceInlineDurArr.push(val["audio_length"]); + allSentenceInlineArr.push(audioFile); + } + + // debugger; + } + let j = 0, + num = 0; + let presentScene = allSentenceArr[0]["scene_num"]; + let flag = true; + for (val of allSentenceArr) { + if (val["scene_num"] != presentScene) { + if (flag) isSceneExtended.push(false); + flag = true; + } + + if (flag && val["audio_type"] == "extended") { + isSceneExtended.push(true); + flag = false; + } + } + audio_flag = true; + }); + console.log("Times", allDialogArraySTartTimes); +} + +function initializeDescriptionsUI() { + let outputHtml = ""; + let listContainer = document.querySelector(".list-container"); + sceneDetails = JSON.parse(listContainer.dataset.sceneDetails); + + for (let i = 0; i < sceneDetails.length; i++) { + outputHtml += renderScenes(sceneDetails[i], i + 1); + } + + $("#ul-scene-list").html(outputHtml); + $("textarea").each(function() { + auto_grow(this); + }); +} + +function renderScenes(currScene, scene_num) { + // console.log({currScene}) + let sceneId = currScene.sc_id; + let sceneNum = currScene.sc_num; + let scStartTime = currScene.start_time; + let scFinishTime = currScene.finish_time; + let sceneOcr = enableAi ? $.trim(currScene.original_ocr) : ""; + let sceneDesc = enableAi ? $.trim(currScene.original_description) : ""; + let scStartTimePretty = millisToMinutesAndSeconds(scStartTime); + let scFinishTimePretty = millisToMinutesAndSeconds(scFinishTime); + let mergeMessage = + "Please merge the scene before generating questions for the next scene"; + let doneSceneBtn = ``; + let getQuestionBtn = ``; + let qaContainer = `
`; + let createMoreQues = ``; + let mergeScene = ``; + let newQuestion = `${getQuestionBtn}
${qaContainer}
${createMoreQues}
${doneSceneBtn}
${mergeScene} ${mergeMessage}`; + + let outputHtml = + `
  • ` + + `` + + "' + + '" + + "
    Scene " + + scene_num + + `` + + scStartTimePretty + + " to " + + scFinishTimePretty + + ' minutes
    Description
    ' + + '
    Text On-Screen" + + '
    ' + + '
    " + + `
    ` + + newQuestion; + + return outputHtml; +} + +const renderQnaList = qnaList => { + qnaList.forEach(qa => { + let { answerId, sceneId, questionId, question, answer } = qa; + let currentContainer = `qna_container:${sceneId}`; + let cont = document.getElementById(currentContainer); + const qaElement = getNewQnA(question, answer, questionId, answerId); + qaElement.dataset.jsonData = JSON.stringify(qa); + cont.appendChild(qaElement); + }); +}; + +/* +created by Vaishali +create Question answer on click of button "addQuestion" +*/ +function getNewQnA(ques, ans, quesId, ansId) { + let qa = document.createElement("div"); + + let del = ``; + qa.innerHTML = `
    + + + + + ${del} +
    `; + return qa; +} + +/* +created by Vaishali +adding eventListener on add question btn click, get question click +*/ +document.getElementById("ul-scene-list").addEventListener("click", e => { + console.log("click event TARGET: ", e.target); + + if (e.target.id.includes("create_question:")) { + let index = e.target.id.indexOf(":") + 1; + let textDescArea = e.target.id.substring(index); + document + .getElementById(`qna_container:${textDescArea}`) + .appendChild(getNewQnA()); + } else if (e.target.id.includes("get_question:")) { + let index = e.target.id.indexOf(":") + 1; + let textDescArea = e.target.id.substring(index); + let descId = "s_d_" + textDescArea; + let desc = document.getElementById(descId).value; + + console.log("descId: ", descId, "val: ", desc); + let rows = desc.split("\n"); + for (var i = 0; i < rows.length; i++) { + console.log("ROWS:", rows[i]); + } + let postJson = { + sentence: rows + }; + + let questionArr = []; + $.post("/us3/question/generate", postJson, function(res) { + rows = res.result; + for (let i = 0; i < rows.length; i++) { + if (rows[i].response.status === "success") { + console.log({ QUESTION: rows[i].response.question }); + questionArr.push(rows[i]); + let response = rows[i].response; + let qnAelement = getNewQnA(response.question, response.answer); + // e.target.nextSibling.nextSibling.appendChild(qnAelement); + + document + .getElementById(`qna_container:${textDescArea}`) + .appendChild(qnAelement); + } + } + console.log("RESULT ", res.result); + console.log({ questionArr }); + }); + } else if (e.target.id.includes("delBtn")) { + console.log("INSIDE DELETEEEE ", e.target); + let [txt, questionId, answerId] = e.target.id.split(":"); + let jsonBody = { + questionId: questionId, + answerId: answerId + }; + console.log("DELETEJSON ", jsonBody); + $.ajax({ + type: "DELETE", + url: "/us3/question/deleteQuestion", + dataType: "json", + async: true, + data: jsonBody, + success: function(response) { + console.log("SUCCESS:", response); + }, + error: function(response, textStatus, errorThrown) { + console.log(errorThrown); + } + }); + } else if (e.target.id === "doneSceneBtn") { + playVideo(); + } else if (e.target.id.includes("merge_scene")) { + let listContainer = document.querySelector(".list-container"); + sceneDetails = JSON.parse(listContainer.dataset.sceneDetails); + + let result = confirm("Do you want to merge current scene with the next?"); + if (result) { + //Logic to delete the item + console.log("inside merge scene"); + let index = e.target.id.indexOf(":") + 1; + let curSceneId = e.target.id.substring(index); + console.log({ curSceneId }); + let curSceneDescElement = document.getElementById("s_d_" + curSceneId); + console.log({ curSceneDescElement }); + + //next Sibling + let nextSibling = document.getElementById(e.target.id).parentElement + .parentElement.nextSibling; + let nextSiblingSceneId = nextSibling.children[0].id; + let indexNextSib = nextSiblingSceneId.indexOf(":") + 1; + nextSiblingSceneId = nextSiblingSceneId.substring(indexNextSib); + let nextSceneData = sceneDetails.filter( + el => el.sc_id === nextSiblingSceneId + )[0]; + let currSceneData = sceneDetails.filter(el => el.sc_id === curSceneId)[0]; + + if (!currSceneData.deletedScene) { + let deletedSceneArr = []; + deletedSceneArr.push(nextSiblingSceneId); + currSceneData.deletedScene = deletedSceneArr; + } else { + if (nextSceneData.deletedScene) { + currSceneData.deletedScene.push(...nextSceneData.deletedScene); + } + + currSceneData.deletedScene.push(nextSiblingSceneId); + } + + console.log({ nextSceneData, currSceneData }); + + let newStartTime = currSceneData.start_time; + let newEndTime = nextSceneData.finish_time; + currSceneData.finish_time = newEndTime; + + document.getElementById( + `time:${curSceneId}` + ).innerHTML = `${millisToMinutesAndSeconds( + newStartTime + )} to ${millisToMinutesAndSeconds(newEndTime)} minutes`; + // console.log({allSceneArr}); + console.log({ nextSiblingSceneId }); + let nextSceneDescElement = document.getElementById( + "s_d_" + nextSiblingSceneId + ); + console.log({ nextSceneDescElement }); + + curSceneDescElement.value += "\n" + nextSceneDescElement.value; + // document.getElementById(curSceneDescElement+'.'+'txt-scene-description').autogrow(); + nextSibling.remove(); + //delete next scene + let jsonBody = { + sceneId: nextSiblingSceneId + }; + + console.log(e.target.id); + console.log({ "Inside merge:": sceneDetails }); + listContainer.dataset.sceneDetails = JSON.stringify(sceneDetails); + updateAll(); + fetchVideoInfo(youtubeId); + // let currentTime = player.getCurrentTime(); + // player.seekTo(currentTime); + } + } +}); + +function millisToMinutesAndSeconds(secs) { + var minutes = Math.floor(secs / 60); + var seconds = ((secs % 60) / 1).toFixed(1); + return minutes + ":" + seconds; +} + +function scrollToPosition(position) { + position--; + $("#ul-scene-list").animate( + { + scrollTop: + $("#ul-scene-list>.newscene-list:eq(" + position + ")").position().top - + $("#ul-scene-list>.newscene-list:eq(0)").position().top + }, + "slow" + ); + let newSceneList = document.querySelectorAll(".newscene-list"); + + newSceneList.forEach(sceneItem => { + sceneItem.firstChild.classList.remove("highlight"); + }); + + newSceneList[position].firstChild.classList.add("highlight"); + + // if(position > 0) { + // newSceneList[position-1].firstChild.classList.remove('highlight'); + // } + console.log({ position }); + console.log({ previouslyHighlightedContainerId }); + previouslyHighlightedContainerId = + "#ul-scene-list>.newscene-list:eq(" + position + ")" + " .scene-container"; +} + +function onYouTubeIframeAPIReady() { + player = new YT.Player("player", { + height: "270", + width: "480", + videoId: youtubeId, + playerVars: { + autoplay: 1, + enablejsapi: 1, + cc_load_policy: 1, + controls: 1, + fs: 0, + iv_load_policy: 3, + modestbranding: 1, + rel: 0, + showinfo: 0, + wmode: "opaque" + }, + events: { + onReady: onPlayerReady, + onStateChange: onPlayerStateChange + } + }); +} + +// function msleep(n){ +// Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, n); +// // Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, n); +// } + +async function onPlayerReady(event) { + pauseVideo(); + player.setVolume(75); + $("#total_time").text(formatTime(player.getDuration())); + updateTime(); + document.getElementById("player").tabIndex = -1; +} + +async function updateTime() { + while (true) { + if (desc_flag) { + let oldTime = videotime; + if (player && player.getCurrentTime) videotime = player.getCurrentTime(); + + if (prevSceneNum >= 0 && videotime - oldTime > 0.3) { + console.log("Forward"); + STATE = 0; + await updateProgress(videotime); // S1 Different func + } else if (oldTime - videotime > 0.3) { + console.log("Rewind"); + STATE = 0; + await updateProgress(videotime); // S1 Different func + } else if (YT_FLAG) YT_FLAG = false; + else if (videotime !== oldTime) { + if (STATE == 0) { + await onProgress(videotime); + } else if (STATE == 1) { + if (YT_FLAG_EXT) { + YT_FLAG_EXT = false; + continue; + } + pauseVideo(); + YT.currentTime = STATE_TIME; + YT_FLAG = true; + console.log("Extended fault"); + + if (currAudio && currAudio !== null) currAudio.pause(); + + STATE = 2; + } else if (STATE == 2) { + pauseVideo(); + YT.currentTime = STATE_TIME; + YT_FLAG = true; + videotime = player.getCurrentTime(); + if (currAudio && currAudio !== null) currAudio.play(); + STATE = 1; + } + } else { + if ( + STATE == 0 && + currAudio && + currAudio !== null && + isPlaying(currAudio) + ) + currAudio.pause(); + } + $("#elapsed_time").text(formatTime(player.getCurrentTime())); + console.log("Time event"); + } + await new Promise(r => setTimeout(r, 100)); + } +} + +// when the time changes, this will be called. + +async function updateProgress(currentTime) { + currentTime = currentTime.toFixed(3); + let currScene = getSceneFromTime(currentTime, allSceneArr); + let sceneNum = currScene.sc_num; + prevSceneNum = sceneNum; + if (currAudio !== null && isPlaying(currAudio)) currAudio.pause(); + + currAudio = null; + player.setVolume(75); +} + +async function onProgress(currentTime) { + // debugger; + currentTime = currentTime.toFixed(3); + let currScene = getSceneFromTime(currentTime); + let sceneId = currScene.sc_id; + console.log(allSceneArr); + let sceneNum = currScene.sc_num; + let sceneEndTime = currScene.finish_time; + + if (prevSceneNum == -1 && first_extend_flag) first_extend_flag = false; + else if (prevSceneNum != -1 && sceneNum != prevSceneNum) { + if (scene_audio_extend_set.has(prevSceneNum)) { + STATE = 1; + pauseVideo(); + let totalAudios = sentenceAudioMap[prevSceneNum]; + let flag = true; + let i = 0; + YT_FLAG = true; + if (flag && currAudio == null) { + currAudio = sentenceAudioMap[prevSceneNum][0]; + player.setVolume(15); + currAudio.play(); + currAudio.onended = function() { + if (player.getPlayerState() == YT.PlayerState.PAUSED) { + STATE = 0; + STATE_TIME = -1; + currAudio = null; + player.setVolume(75); + playVideo(); + } + }; + // If multiple extended clips exist for same timestamp + + // while(i setTimeout(r, 100)); + + // } + + prevSceneNum = sceneNum; + STATE = 1; + STATE_CHANGE = 1; + console.log("STATE: EXTENDED"); + + STATE_TIME = player.getCurrentTime(); + } + prevSceneNum = sceneNum; + } + prevSceneNum = sceneNum; + } else if (prevSceneNum == -1) prevSceneNum = sceneNum; + else if (currAudio !== null && !isPlaying(currAudio)) currAudio.play(); + // logic for inline + else { + let res = isInlineAudio(currentTime); + let index = res[1]; + if (res[0] && currAudio == null && index >= 0) { + currAudio = allSentenceInlineArr[index]; + let diff = res[2]; + currAudio.currentTime = diff; + player.setVolume(15); + currAudio.play(); + currAudio.onended = function() { + currAudio = null; + player.setVolume(75); + }; + } + } +} + +function onPlayerStateChange(event) { + let playerState = event.data; + switch (playerState) { + case YT.PlayerState.PLAYING: + console.log("i was called: playing state"); + $("#btn-play-pause").html( + '' + ); + break; + case YT.PlayerState.PAUSED: + $("#btn-play-pause").html( + '' + ); + timeAtWhichLastPaused = player.getCurrentTime(); + console.log(timeAtWhichLastPaused); + console.log("i was called: paused state"); + break; + case YT.PlayerState.UNSTARTED: + case YT.PlayerState.ENDED: + case YT.PlayerState.CUED: + $("#btn-play-pause").html( + '' + ); + timeAtWhichLastPaused = player.getCurrentTime(); + console.log("i was called: cued state"); + break; + } +} + +function playVideo() { + player.playVideo(); +} + +function pauseVideo() { + console.log("This is the file"); + player.pauseVideo(); +} + +function stopVideo() { + player.stopVideo(); +} + +/* +created by Vaishali +iterating over all QnAcontainers, saving entire info for the ques and answers that already exists +in the DB, sending new qnAs +*/ +const getNewQnAData = () => { + const qnaData = {}; + var allQAContainer = $('div[id^="qna_container:"]'); + + for (let i = 0; i < allQAContainer.length; i++) { + const qaList = allQAContainer[i].children; + for (let childIndex = 0; childIndex < qaList.length; childIndex++) { + console.log("id of parent", allQAContainer[i].id.substring(14)); + let sceneId = allQAContainer[i].id.substring(14); + const currentElement = qaList[childIndex]; + var question = currentElement.children[0].children[1].value; + var answer = currentElement.children[0].children[3].value; + console.log({ question, answer }); + + qnaData[sceneId] = qnaData[sceneId] || []; + if (currentElement.dataset.jsonData) { + const tempData = JSON.parse(currentElement.dataset.jsonData); //saving coming json data for the exisiting qnA + qnaData[sceneId].push({ + question, + answer, + answerId: tempData.answerId, + questionId: tempData.questionId + }); + } else { + qnaData[sceneId].push({ videoId: youtubeId, question, answer }); //pushing only questio and answer value to the exisiting qnA. + } + } + } + return qnaData; +}; + +async function updateAll(shoudShowAlert) { + let postJson = fetchJsonBodyForAllTextFields(userId); + const qnaData = getNewQnAData(); + postJson.qnaData = qnaData; + postJson.videoId = youtubeId; + console.log({ HERE: postJson }); + try { + await $.post("/saveAiDescription", postJson, function(response) { + if (shoudShowAlert) { + if (response.success) { + alert( + "Submitted Successfully\nYou will be redirected to the Video List Page" + ); + window.history.back(); + } else { + console.log(response); + alert("Update Status: " + response.success); + } + } + }); + } catch (e) { + console.log(e); + response.send(e); + } +} + +//added start time, end time and has Ai for saving in descrption table +function fetchJsonBodyForAllTextFields(userId) { + let listContainer = document.querySelector(".list-container"); + sceneDetails = JSON.parse(listContainer.dataset.sceneDetails); + + let parentJsonBody = {}; + let sceneArrJson = []; + + console.log({ "INSIDE fetchJsonBodyForAllTextFields ": sceneDetails }); + let currScene; + for (let i = 0; i < sceneDetails.length; i++) { + currScene = sceneDetails[i]; + let currSceneId = currScene.sc_id; + let currSceneDescTextId = "#s_d_" + currSceneId; + let currSceneOcrTextId = "#s_o_" + currSceneId; + let sceneJsonObj = {}; + sceneJsonObj.scene_id = currSceneId; + let csTestbox = $(currSceneDescTextId); + if (csTestbox && csTestbox.val()) { + sceneJsonObj.modified_description = csTestbox.val().trim(); + sceneJsonObj.modified_ocr = $(currSceneOcrTextId) + .val() + .trim(); + sceneJsonObj.start_time = currScene.start_time; + sceneJsonObj.end_time = currScene.finish_time; + sceneJsonObj.has_ai = enableAi; + sceneJsonObj.deletedScene = currScene.deletedScene; + sceneArrJson.push(sceneJsonObj); + } + } + + parentJsonBody.user_id = userId; + parentJsonBody.scene_arr = sceneArrJson; + parentJsonBody.time_taken = timeElapsedOnPage; + + console.log({ parentJsonBody }); + return parentJsonBody; +} + +async function updateMainTables() { + let postJson = fetchJsonBodyForAllTextFields2(allSceneArr, allBlockArr); + + console.log({ postJson }); + await $.post("/updateBlockAndScene", postJson, function(response) { + if (response.success) { + window.alert("Updated Successfully"); + } else { + window.alert("Update Failed"); + } + }); +} + +function fetchJsonBodyForAllTextFields2(allSceneArr, allBlockArr, userId) { + let parentJsonBody = {}; + let sceneArrJson = []; + let blockArrJson = []; + for (let i = 0; i < allSceneArr.length; i++) { + let currScene = allSceneArr[i]; + let currSceneId = currScene.sc_id; + let currSceneDescTextId = "#s_d_" + currSceneId; + let currSceneOcrTextId = "#s_o_" + currSceneId; + let sceneJsonObj = {}; + sceneJsonObj.scene_id = currSceneId; + sceneJsonObj.original_description = $(currSceneDescTextId) + .val() + .trim(); + sceneJsonObj.original_ocr = $(currSceneOcrTextId) + .val() + .trim(); + sceneArrJson.push(sceneJsonObj); + } + for (let i = 0; i < allBlockArr.length; i++) { + let currBlock = allBlockArr[i]; + let currBlockId = currBlock.block_id; + let currBlockDescTextId = "#b_d_" + currBlockId; + let blockJsonObj = {}; + blockJsonObj.block_id = currBlockId; + blockJsonObj.original_description = $(currBlockDescTextId) + .val() + .trim(); + blockArrJson.push(blockJsonObj); + } + parentJsonBody.scene_arr = sceneArrJson; + parentJsonBody.block_arr = blockArrJson; + return parentJsonBody; +} + +function populateSceneIdToListPositionMap(sceneDetailsArr) { + let listPositionIndex = 0; + let sceneListPositionMap = new Map(); + + for (let i = 0; i < sceneDetailsArr.length; i++) { + let currSceneId = sceneDetailsArr[i].sc_id; + sceneListPositionMap.set(currSceneId, ++listPositionIndex); //sceneToBlockMap + } + return sceneListPositionMap; +} + +function populateSceneIdToListPosition(blocksJsonArr) { + let listPositionIndex = 0; + let sceneListPositionMap = new Map(); + for (let i = 0; i < blocksJsonArr.length; i++) { + listPositionIndex++; + let currBlock = blocksJsonArr[i]; + let currBlockId = currBlock.block_id; + let childSceneArr = currBlock.child_scenes; + for (let j = 0; j < childSceneArr.length; j++) { + listPositionIndex++; + let currSceneId = childSceneArr[j].sc_id; + sceneListPositionMap.set(currSceneId, listPositionIndex); //sceneToBlockMap + if (j == childSceneArr.length - 1) { + sceneToBlockMap.set(currSceneId, currBlockId); + } + } + listPositionIndex++; + } + desc_flag = true; + return sceneListPositionMap; +} + +function getSceneFromTime(currentTime) { + let listContainer = document.querySelector(".list-container"); + console.log("This: " + listContainer.dataset); + sceneDetails = JSON.parse(listContainer.dataset.sceneDetails); + + let startTimesArr = []; + for (let i = 0; i < sceneDetails.length; i++) { + let startTime = parseFloat(sceneDetails[i].start_time); + startTimesArr.push(startTime); + } + let sceneIndex = getIndexOfInputArrForTargetVal(currentTime, startTimesArr); + return sceneDetails[sceneIndex]; +} + +function getSceneFromTime2(currentTime, sceneArr) { + let startTimesArr = []; + for (let i = 0; i < sceneArr.length; i++) { + let startTime = parseFloat(sceneArr[i].start_time); + startTimesArr.push(startTime); + } + let sceneIndex = getIndexOfInputArrForTargetVal(currentTime, startTimesArr); + return sceneArr[sceneIndex]; +} +// Convert to Binary search +function getDialogFromTime(currentTime) { + let index = allDialogArraySTartTimes.length - 1; + for (let i = 0; i < allDialogArraySTartTimes.length; i++) { + if (Number(currentTime) < allDialogArraySTartTimes[i]) { + index = i - 1; + console.log("************************************** " + i); + break; + } + } + // console.log("Dialog times: i: "+i+" currentTime: "+currentTime); + console.log( + "cTime: " + + typeof currentTime + + " typeof Dialog: " + + typeof allDialogArraySTartTimes[4] + + " i: " + + index + ); + if (Number(currentTime) < allDialogArrayEndTimes[index]) { + console.log("##########################################################"); + return true; + } else return false; +} + +//Returns index of array item whose value is equal-to or lesser-than(also nearest-in-value) the target (binary search algorithm) +function getIndexOfInputArrForTargetVal(target, inputArr) { + let start = 0; + let end = inputArr.length - 1; + let index = -1; + while (start <= end) { + let mid = Math.floor((start + end) / 2); + if (inputArr[mid] == target) { + index = mid; + break; + } else if (inputArr[mid] < target) { + index = mid; + start = mid + 1; + } else { + end = mid - 1; + } + } + return index; +} + +function isInlineAudio(target) { + let inputArr = allSentenceInlineStartTimeArr; + let start = 0; + let end = inputArr.length - 1; + let index = -1; + while (start <= end) { + let mid = Math.floor((start + end) / 2); + if (inputArr[mid] == target) { + index = mid; + break; + } else if (inputArr[mid] < target) { + index = mid; + start = mid + 1; + } else { + end = mid - 1; + } + } + console.log( + "inside inline: ", + target, + " and ", + allSentenceInlineStartTimeArr[index] + allSentenceInlineDurArr[index] + ); + if ( + index < 0 || + target > + allSentenceInlineStartTimeArr[index] + allSentenceInlineDurArr[index] + ) + return [false, index, 0]; + // may return the difference of current time and inline start time + else return [true, index, target - allSentenceInlineStartTimeArr[index]]; +} + +function getUrlVars() { + var vars = {}; + var parts = window.location.href.replace(/[?&]+([^=&]+)=([^&]*)/gi, function( + m, + key, + value + ) { + vars[key] = value; + }); + return vars; +} + +function auto_grow(element) { + element.style.height = "5px"; + element.style.height = element.scrollHeight + "px"; +} + +function resetVideoTo(time) { + let timeInSecs = parseFloat(time); + player.seekTo(timeInSecs); + player.playVideo(); +} + +function formatTime(time) { + var minutes = Math.floor(time / 60); + var seconds = time - minutes * 60; + seconds = seconds < 10 ? "0" + seconds : seconds; + var s = parseFloat(seconds).toFixed(1); + return minutes + ":" + s; +} + +function formatTimeForTimer(time) { + var minutes = Math.floor(time / 60); + var seconds = time - minutes * 60; + seconds = seconds < 10 ? "0" + seconds : seconds; + var s = parseFloat(seconds).toFixed(0); + return minutes + ":" + s; +} + +function onPlayPauseButtonClick() { + let playerState = player.getPlayerState(); + switch (playerState) { + case YT.PlayerState.PLAYING: + pauseVideo(); + break; + case YT.PlayerState.PAUSED: + case YT.PlayerState.UNSTARTED: + case YT.PlayerState.ENDED: + case YT.PlayerState.CUED: + playVideo(); + break; + } +} + +function populateAllSceneArrAndAllBlockArr(blocksJsonArr) { + for (let i = 0; i < blocksJsonArr.length; i++) { + let currBlock = blocksJsonArr[i]; + allBlockArr.push(currBlock); + let childSceneArr = currBlock.child_scenes; + allSceneArr.push.apply(allSceneArr, childSceneArr); + } +} + +function isPlaying(audioElement) { + if (audioElement != null) return !audioElement.paused; +} diff --git a/public/javascripts/audioDescriptionUserStudy.js b/public/javascripts/audioDescriptionUserStudy.js new file mode 100644 index 0000000..e8a0e29 --- /dev/null +++ b/public/javascripts/audioDescriptionUserStudy.js @@ -0,0 +1,593 @@ +let player; +let youtubeId = ""; +let videotime = 0; +let timeupdater = null; +let sceneIdToListPositionMap = new Map(); +let sceneToBlockMap = new Map(); +// let userId = "d8fcfd52-b0b3-4969-bf03-1dc54b681b5d"; +let userId = "f03cb948-474b-4084-9192-650ba62d396b"; +let volunteerId; +let prevSceneNum = -1; +let lastSceneNum = -1; +let timeAtWhichLastPaused = -1; +let previouslyHighlightedContainerId = "#doesnotexist"; +let enableAi = false; +let baselineDesc = "and Baseline Descriptions"; +let autoPauseAtEachScene = true; +let timeElapsedOnPage = 0; +let isTimerPaused = false; +let allSceneArr = []; +let allBlockArr = []; +let audioArr = []; +let startTimesArr = []; +let allDialogArraySTartTimes = []; +let allDialogArrayEndTimes = []; + +let allSentenceStartTimes = []; +let allSentenceEndTimes = []; +let isSceneExtended = []; // Required to check if a scene has to be auto-paused to play the extended audio +let allSentenceAudioType = []; +let allSentenceSceneNum = []; +let scene_audio_extend_set = new Set(); +let sentenceAudioMap = {}; +let allSentenceInlineStartTimeArr = []; +let allSentenceInlineDurArr = []; +let allSentenceInlineArr = []; +let allSentenceExtendedArr = []; +let allSentenceExtendedAudioArr = []; + +let currAudio = null; +let currAudioTime; +let wasCurrAudioPaused = false; +let STATE = 0; +let YT_STATE = 0; +let STATE_TIME = -1; +let STATE_CHANGE = 0; +let YT_FLAG = false; +let YT_FLAG_Dial = false; +let YT_FLAG_EXT = false; +let first_extend_flag = true; +let desc_flag = false; +let audio_flag = false; +let noOfQuestionsAsked = 0; +let isTutorial = false; + +$(document).ready(function() { + $("#btn-video-breakpoint-toggle").prop("checked", true); + volunteerId = getUrlVars()["u"]; + // userId = getUrlVars()["u"]; + describerId = getUrlVars()["did"]; + userId = + describerId == "1" + ? "f03cb948-474b-4084-9192-650ba62d396b" + : "c833874d-e2f1-45b3-afd1-16ac6d7fa0b4"; + enableAi = getUrlVars()["ai"] == "yes" ? true : false; + // if(!enableAi){ + // userId = ''; + // baselineDesc = 'Descriptions'; + // } + // document.getElementById("page-title").innerHTML = "On Demand " + baselineDesc; + + let href = location.href; + const lastPathSegment = new URL(href).pathname.split("/").pop(); + youtubeId = lastPathSegment; + fetchVideoInfo(youtubeId); + + $("#btn-play-pause").click(function() { + onPlayPauseButtonClick(); + }); + $("#btn-timer").click(function() { + isTimerPaused = !isTimerPaused; + if (isTimerPaused) { + $("#btn-timer").attr("src", "/images/play_white.png"); + } else { + $("#btn-timer").attr("src", "/images/pause_white.png"); + } + }); + setInterval(incrementSeconds, 1000); +}); + +function incrementSeconds() { + if (!isTimerPaused) { + timeElapsedOnPage += 1; + $("#txt-timeElaspsed").html(formatTimeForTimer(timeElapsedOnPage)); + } +} + +function fetchVideoInfo(youtubeId) { + // WebSocketTest(); + console.log(userId); + // let targetUrl = '/fetchVideoData' + '?videoid=' + youtubeId; + let targetUrl = `/fetchVideoData?videoid=${youtubeId}&userId=${userId}`; + $.get(targetUrl, function(response) { + let vidJson = response; + let vidTitle = vidJson.title; + $("#txtVidtitle").html(vidTitle); + desc_flag = true; + }); + + let target_url_sentence = + "/getSentences" + "?videoId=" + youtubeId + "&userId=" + userId; + $.get(target_url_sentence, function(response) { + // KeyFramesArr = response; + // debugger; + console.log("sentence start times" + response[0]); + allSentenceArr = response; + let prevSceneNo = -1; + for (val of response) { + allSentenceStartTimes.push(val["audio_start_time"]); + // allSentenceEndTimes.push(val['audio_end_time']); + allSentenceAudioType.push(val["audio_type"]); + allSentenceSceneNum.push(val["scene_num"]); + // let sceneId = val['scene_id']; + let sceneNum = val["scene_num"]; + if (sceneNum != prevSceneNo) { + startTimesArr.push(val["start_time"]); + prevSceneNo = sceneNum; + } + let randomNum = Math.random(); + // let descType = val['has_ai'] ? 'ai' : 'non_ai'; + let descType = "ai"; + // let sentNumArr = val['sentence_id'].split('_'); + // let sentNum = sentNumArr[sentNumArr.length - 1]; + let audioType = val["audio_type"]; + let path = val["audio_path"]; + // let path = val['audio_path'].split('/'); + // let path_ext = path[path.length - 1]; + // let src = `../audio/${youtubeId}/${descType}/${userId}/${sceneNum}/new/${path_ext}` + '?random='+randomNum; + let src = path + "?random=" + randomNum; + console.log(src); + let audioFile = new Audio(src); + // audioArr.push(audioFile); + + if ( + !scene_audio_extend_set.has(val["scene_num"]) && + val["audio_type"] == "extended" + ) { + scene_audio_extend_set.add(val["scene_num"]); + //create audio object + sentenceAudioMap[val["scene_num"]] = [audioFile]; + allSentenceExtendedArr.push(val["audio_start_time"]); + allSentenceExtendedAudioArr.push(audioFile); + } else if (val["audio_type"] == "extended") { + // create audio object + sentenceAudioMap[val["scene_num"]].push(audioFile); + allSentenceExtendedArr.push(val["audio_start_time"]); + allSentenceExtendedAudioArr.push(audioFile); + } else if (val["audio_type"] == "inline") { + allSentenceInlineStartTimeArr.push(val["audio_start_time"]); + allSentenceInlineDurArr.push(val["audio_length"]); + allSentenceInlineArr.push(audioFile); + } + + // debugger; + } + let j = 0, + num = 0; + let presentScene = allSentenceArr[0]["scene_num"]; + let flag = true; + for (val of allSentenceArr) { + if (val["scene_num"] != presentScene) { + if (flag) isSceneExtended.push(false); + flag = true; + } + + if (flag && val["audio_type"] == "extended") { + isSceneExtended.push(true); + flag = false; + } + } + audio_flag = true; + }); + console.log("Times", allDialogArraySTartTimes); +} + +function millisToMinutesAndSeconds(secs) { + var minutes = Math.floor(secs / 60); + var seconds = ((secs % 60) / 1).toFixed(1); + return minutes + ":" + seconds; +} + +function onYouTubeIframeAPIReady() { + let href = location.href; + const youtubeIframeId = new URL(href).pathname.split("/").pop(); + + player = new YT.Player("player", { + height: "270", + width: "480", + videoId: youtubeIframeId, + playerVars: { + autoplay: 1, + enablejsapi: 1, + cc_load_policy: 1, + controls: 0, + disablekb: 1, + fs: 0, + iv_load_policy: 3, + modestbranding: 1, + rel: 0, + showinfo: 0, + wmode: "opaque" + }, + events: { + onReady: onPlayerReady, + onStateChange: onPlayerStateChange + } + }); +} + +async function onPlayerReady(event) { + pauseVideo(); + player.setVolume(75); + $("#total_time").text(formatTime(player.getDuration())); + updateTime(); + document.getElementById("player").tabIndex = -1; +} + +async function updateTime() { + while (true) { + if (desc_flag) { + var oldTime = videotime; + if (player && player.getCurrentTime) { + videotime = player.getCurrentTime(); + // console.log("No state change"); + } + // if(Math.floor(videotime/30) <= noOfQuestionsAsked) + // STATE = 0; + + // if(Math.floor(videotime/30) > noOfQuestionsAsked){ + // if(player.getPlayerState() == YT.PlayerState.PLAYING){ + // pauseVideo(); + // await checkQuestion(); + // } + // } + + // else if(prevSceneNum >= 0 && videotime - oldTime > 0.3) { + if (prevSceneNum >= 0 && videotime - oldTime > 0.3) { + console.log("Forward"); + STATE = 0; + await updateProgress(videotime); // S1 Different func + } else if (oldTime - videotime > 0.3) { + console.log("Rewind"); + STATE = 0; + await updateProgress(videotime); // S1 Different func + } else if (YT_FLAG) YT_FLAG = false; + else if (videotime !== oldTime) { + console.log( + "Playing: oldtime: " + + oldTime + + " videotime: " + + videotime + + " State time: " + + STATE_TIME + ); // S2 Different func + if (STATE == 0) { + // currAudio.play(); + await onProgress(videotime); + } else if (STATE == 1) { + if (YT_FLAG_EXT) { + YT_FLAG_EXT = false; + continue; + } + + pauseVideo(); + YT.currentTime = STATE_TIME; + YT_FLAG = true; + console.log("Extended fault"); + if (currAudio && currAudio !== null) currAudio.pause(); + // currAudio.play(); + console.log("Current Audio is paused: 0" + STATE_TIME); + STATE = 2; + } else if (STATE == 2) { + pauseVideo(); + YT.currentTime = STATE_TIME; + YT_FLAG = true; + videotime = player.getCurrentTime(); + if (currAudio && currAudio !== null) currAudio.play(); + STATE = 1; + } + } else { + // console.log("Paused"); // modify currAudio.onended func + + if ( + STATE == 0 && + currAudio && + currAudio !== null && + isPlaying(currAudio) + ) + currAudio.pause(); + } + $("#elapsed_time").text(formatTime(player.getCurrentTime())); + console.log("Time event"); + // msleep(100); + } + await new Promise(r => setTimeout(r, 100)); + } +} + +// when the time changes, this will be called. + +async function updateProgress(currentTime) { + currentTime = currentTime.toFixed(3); + let currScene = getIndexFromExtended(currentTime); + + // let sceneNum = currScene.sc_num; + + prevSceneNum = currScene; + + if (currAudio !== null && isPlaying(currAudio)) currAudio.pause(); + currAudio = null; + player.setVolume(75); +} + +async function onProgress(currentTime) { + // debugger; + currentTime = currentTime.toFixed(3); + // let currScene = getSceneFromTime(currentTime); + let currScene = getIndexFromExtended(currentTime); + // console.log(allSceneArr); + // let sceneNum = currScene.sc_num; + let sceneNum = currScene; + + if (prevSceneNum == -1 && first_extend_flag) { + first_extend_flag = false; + } + + // else if((prevSceneNum != -1 && sceneNum != prevSceneNum)) { + else if (sceneNum != -1 && sceneNum != prevSceneNum) { + if (true || scene_audio_extend_set.has(prevSceneNum)) { + STATE = 1; + pauseVideo(); + let totalAudios = sentenceAudioMap[prevSceneNum]; + let flag = true; + let i = 0; + YT_FLAG = true; + if (flag && currAudio == null) { + // currAudio = sentenceAudioMap[prevSceneNum][0]; + currAudio = allSentenceExtendedAudioArr[currScene]; + console.log( + "Will play now!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + ); + player.setVolume(15); + currAudio.play(); + currAudio.onended = function() { + console.log("player state is: " + player.getPlayerState()); + if (player.getPlayerState() == YT.PlayerState.PAUSED) { + console.log("I was called"); + STATE = 0; + STATE_TIME = -1; + currAudio = null; + player.setVolume(75); + playVideo(); + } + }; + + prevSceneNum = sceneNum; + + // YT_FLAG_EXT = true; + STATE = 1; + STATE_CHANGE = 1; + console.log("STATE: EXTENDED"); + + STATE_TIME = player.getCurrentTime(); + // debugger; + // prevSceneNum = sceneNum-1; + } + prevSceneNum = sceneNum; + } + prevSceneNum = sceneNum; + } + + // else if(prevSceneNum == -1) + // prevSceneNum = sceneNum; + else if (currAudio !== null && !isPlaying(currAudio)) currAudio.play(); + // logic for inline + else { + let res = isInlineAudio(currentTime); + let index = res[1]; + if (res[0] && currAudio == null && index >= 0) { + currAudio = allSentenceInlineArr[index]; + let diff = res[2]; + currAudio.currentTime = diff; + player.setVolume(15); + currAudio.play(); + currAudio.onended = function() { + currAudio = null; + console.log("Test1: ", index); + player.setVolume(75); + }; + } + } + // else{ + // // YT_FLAG_Dial = false; + // if(currAudio!==null && !isPlaying(currAudio) && !YT_FLAG_Dial) + // currAudio.play(); + // } +} + +function onPlayerStateChange(event) { + let playerState = event.data; + switch (playerState) { + case YT.PlayerState.PLAYING: + // audio_clip.pause(); + // isChatbotInProcess = false; + // isQuestionInProcess = false; + console.log("i was called: playing state"); + $("#btn-play-pause").html( + '' + ); + break; + case YT.PlayerState.PAUSED: + $("#btn-play-pause").html( + '' + ); + timeAtWhichLastPaused = player.getCurrentTime(); + console.log(timeAtWhichLastPaused); + console.log("i was called: paused state"); + break; + case YT.PlayerState.UNSTARTED: + case YT.PlayerState.ENDED: + if (scene_audio_extend_set.has(sceneNum)) { + // STATE = 1; + // pauseVideo(); + // let totalAudios = sentenceAudioMap[prevSceneNum]; + let flag = true; + // let i=0; + // YT_FLAG = true; + if (flag && currAudio == null) { + currAudio = sentenceAudioMap[sceneNum][0]; + console.log( + "Will play now!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + ); + player.setVolume(15); + currAudio.play(); + currAudio.onended = function() { + console.log("player state is: " + player.getPlayerState()); + }; + } + } + case YT.PlayerState.CUED: + $("#btn-play-pause").html( + '' + ); + timeAtWhichLastPaused = player.getCurrentTime(); + console.log("i was called: cued state"); + break; + } +} + +function playVideo() { + player.playVideo(); +} + +function pauseVideo() { + console.log("This is the file"); + player.pauseVideo(); +} + +function stopVideo() { + player.stopVideo(); +} + +function getSceneFromTime(currentTime) { + let listContainer = document.querySelector(".list-container"); + console.log("This: " + listContainer.dataset); + sceneDetails = JSON.parse(listContainer.dataset.sceneDetails); + + // let startTimesArr = []; + // for(let i=0; i< sceneDetails.length; i++) { + // let startTime = parseFloat(sceneDetails[i].start_time); + // startTimesArr.push(startTime); + // } + let sceneIndex = getIndexOfInputArrForTargetVal(currentTime, startTimesArr); + return sceneDetails[sceneIndex]; +} + +function getIndexFromExtended(currentTime) { + let index = getIndexOfInputArrForTargetVal( + currentTime, + allSentenceExtendedArr + ); + console.log(index); + return index; +} + +//Returns index of array item whose value is equal-to or lesser-than(also nearest-in-value) the target (binary search algorithm) +function getIndexOfInputArrForTargetVal(target, inputArr) { + let start = 0; + let end = inputArr.length - 1; + let index = -1; + while (start <= end) { + let mid = Math.floor((start + end) / 2); + if (inputArr[mid] == target) { + index = mid; + break; + } else if (inputArr[mid] < target) { + index = mid; + start = mid + 1; + } else { + end = mid - 1; + } + } + return index; +} + +function isInlineAudio(target) { + let inputArr = allSentenceInlineStartTimeArr; + let start = 0; + let end = inputArr.length - 1; + let index = -1; + while (start <= end) { + let mid = Math.floor((start + end) / 2); + if (inputArr[mid] == target) { + index = mid; + break; + } else if (inputArr[mid] < target) { + index = mid; + start = mid + 1; + } else { + end = mid - 1; + } + } + console.log( + "inside inline: ", + target, + " and ", + allSentenceInlineStartTimeArr[index] + allSentenceInlineDurArr[index] + ); + if ( + index < 0 || + target > + allSentenceInlineStartTimeArr[index] + allSentenceInlineDurArr[index] + ) + return [false, index, 0]; + // may return the difference of current time and inline start time + else return [true, index, target - allSentenceInlineStartTimeArr[index]]; +} + +function getUrlVars() { + var vars = {}; + var parts = window.location.href.replace(/[?&]+([^=&]+)=([^&]*)/gi, function( + m, + key, + value + ) { + vars[key] = value; + }); + return vars; +} + +function formatTime(time) { + var minutes = Math.floor(time / 60); + var seconds = time - minutes * 60; + seconds = seconds < 10 ? "0" + seconds : seconds; + var s = parseFloat(seconds).toFixed(1); + return minutes + ":" + s; +} + +function formatTimeForTimer(time) { + var minutes = Math.floor(time / 60); + var seconds = time - minutes * 60; + seconds = seconds < 10 ? "0" + seconds : seconds; + var s = parseFloat(seconds).toFixed(0); + return minutes + ":" + s; +} + +function onPlayPauseButtonClick() { + let playerState = player.getPlayerState(); + switch (playerState) { + case YT.PlayerState.PLAYING: + pauseVideo(); + break; + case YT.PlayerState.PAUSED: + case YT.PlayerState.UNSTARTED: + case YT.PlayerState.ENDED: + case YT.PlayerState.CUED: + playVideo(); + break; + } +} + +function isPlaying(audioElement) { + if (audioElement != null) return !audioElement.paused; +} diff --git a/public/javascripts/chatRoom.js b/public/javascripts/chatRoom.js index b6d7099..656fa1d 100644 --- a/public/javascripts/chatRoom.js +++ b/public/javascripts/chatRoom.js @@ -1,147 +1,166 @@ -const socket = io('http://localhost:5001'); +const socket = io("http://localhost:5001"); let url = location.search.substring(1); -const roomName = new URLSearchParams(url).get('name'); -const name = new URLSearchParams(url).get('user'); -const userType = new URLSearchParams(url).get('type'); -const videoId = new URLSearchParams(url).get('videoId'); -console.log('START PAGE'); -console.log({url}); -console.log({roomName}); -console.log({userName: name}); -const ANSWER = 'answer'; -const QUESTION = 'question'; -const INFO = 'info'; - +const roomName = new URLSearchParams(url).get("name"); +const name = new URLSearchParams(url).get("user"); +const userType = new URLSearchParams(url).get("type"); +const videoId = new URLSearchParams(url).get("videoId"); +console.log("START PAGE"); +console.log({ url }); +console.log({ roomName }); +console.log({ userName: name }); +const ANSWER = "answer"; +const QUESTION = "question"; +const INFO = "info"; let questions = {}; -let messageForm = document.getElementById('send-container'); -let messageInput = document.getElementById('message-input'); -let messageContainer = document.getElementById('message-container'); -let recordButton = document.getElementById('record-button'); -let heading = document.getElementById('heading'); -let submitButton = document.getElementById('submit-button'); +let messageForm = document.getElementById("send-container"); +let messageInput = document.getElementById("message-input"); +let messageContainer = document.getElementById("message-container"); +let recordButton = document.getElementById("record-button"); +let heading = document.getElementById("heading"); +let submitButton = document.getElementById("submit-button"); let totalQuestions = 0; - -if(userType === 'admin'){ - recordButton.style.display = "none"; - submitButton.style.display = "none"; +if (userType === "admin") { + recordButton.style.display = "none"; + submitButton.style.display = "none"; } //Video -var tag = document.createElement('script'); +var tag = document.createElement("script"); tag.src = "https://www.youtube.com/iframe_api"; -var firstScriptTag = document.getElementsByTagName('script')[0]; +var firstScriptTag = document.getElementsByTagName("script")[0]; firstScriptTag.parentNode.insertBefore(tag, firstScriptTag); firstScriptTag.append(`

    hello

    `); var player; function onYouTubeIframeAPIReady() { - player = new YT.Player('player', { - height: '390', - width: '640', + player = new YT.Player("player", { + height: "390", + width: "640", videoId: videoId, events: { - 'onReady': onPlayerReady, - 'onStateChange': onPlayerStateChange + onReady: onPlayerReady, + onStateChange: onPlayerStateChange } - }); + }); } function onPlayerReady(event) { - event.target.playVideo(); + event.target.playVideo(); + document.getElementById("player").tabIndex = -1; } -if(name){ - appendMessage('You joined'); - socket.emit('new-user', roomName, name); +if (name) { + appendMessage("You joined"); + socket.emit("new-user", roomName, name); } -socket.on('chat-message', data=>{ - console.log(data); - if(data.type == QUESTION){ - // speak(data.message); - appendQnA(data); - }else if(userType === 'admin' && data.type == INFO){ - appendMessage(data.userId + ':'+ data.message); - }else if(data.type == ANSWER){ - let message = data.name + ' said ' + data.message; - speak(message); - updateAnswer(data); - speak(' Please rate the response'); - } -}) - -socket.on('user-connected', name=>{ - appendMessage(`${name} joined`); -}) - -socket.on('user-disconnected', name=>{ - appendMessage(`${name} disconnected`); -}) -submitButton.addEventListener('click', (e)=>{ - console.log({event: e.target.id}); - if(totalQuestions < 5){ - console.log('inside submit-button'); - speak('Please submit atleast 5 questions'); - }else{ - speak('Thank you for completing the session'); - } - e.stopPropagation(); -}) - -messageContainer.addEventListener('click', (e)=>{ - console.log({event: e.target.id}); - if(e.target.id.includes('sendButton:')){ - console.log('inside send button'); - let index = e.target.id.indexOf(':')+1; - let textDescArea = e.target.id.substring(index); - console.log({textDescArea}); - let answerElement = document.getElementById(`answer:${textDescArea}`); - console.log('ansElement: ', answerElement.value); - socket.emit('send-chat-message', roomName, answerElement.value, textDescArea, name, ANSWER); - } - e.stopPropagation(); -}) - -function appendQnA(data){ - totalQuestions++; - let messageElement = document.createElement('div'); - // messageElement.innerText = message; - // - messageElement.innerHTML = `
    +socket.on("chat-message", data => { + console.log(data); + if (data.type == QUESTION) { + // speak(data.message); + appendQnA(data); + } else if (userType === "admin" && data.type == INFO) { + appendMessage(data.userId + ":" + data.message); + } else if (data.type == ANSWER) { + let message = data.name + " said " + data.message; + speak(message); + updateAnswer(data); + speak(" Please rate the response"); + } +}); + +socket.on("user-connected", name => { + appendMessage(`${name} joined`); +}); + +socket.on("user-disconnected", name => { + appendMessage(`${name} disconnected`); +}); +submitButton.addEventListener("click", e => { + console.log({ event: e.target.id }); + if (totalQuestions < 5) { + console.log("inside submit-button"); + speak("Please submit atleast 5 questions"); + } else { + speak("Thank you for completing the session"); + } + e.stopPropagation(); +}); + +messageContainer.addEventListener("click", e => { + console.log({ event: e.target.id }); + if (e.target.id.includes("sendButton:")) { + console.log("inside send button"); + let index = e.target.id.indexOf(":") + 1; + let textDescArea = e.target.id.substring(index); + console.log({ textDescArea }); + let answerElement = document.getElementById(`answer:${textDescArea}`); + console.log("ansElement: ", answerElement.value); + socket.emit( + "send-chat-message", + roomName, + answerElement.value, + textDescArea, + name, + ANSWER + ); + } + e.stopPropagation(); +}); + +function appendQnA(data) { + totalQuestions++; + let messageElement = document.createElement("div"); + // messageElement.innerText = message; + // + messageElement.innerHTML = `
    - + - - ${userType == 'admin' ? `` : '' } - ${userType == 'participant' ? `Rating: ` : ''} + + ${ + userType == "admin" + ? `` + : "" + } + ${ + userType == "participant" + ? `Rating: ` + : "" + }
    `; - messageContainer.append(messageElement); + messageContainer.append(messageElement); } -function appendMessage(message){ - let messageElement = document.createElement('div'); - messageElement.innerText = message; - messageContainer.append(messageElement); +function appendMessage(message) { + let messageElement = document.createElement("div"); + messageElement.innerText = message; + messageContainer.append(messageElement); } - -function updateAnswer(data){ - console.log('inside updateAnswer:', data); - let answerId = 'answer:' + data.questionId; - document.getElementById(answerId).innerText = data.message; +function updateAnswer(data) { + console.log("inside updateAnswer:", data); + let answerId = "answer:" + data.questionId; + document.getElementById(answerId).innerText = data.message; } // let supportMsg = document.getElementById('msg'); - //Speech Recognition -window.SpeechRecognition = window.webkitSpeechRecognition || window.SpeechRecognition; +window.SpeechRecognition = + window.webkitSpeechRecognition || window.SpeechRecognition; // if ('SpeechRecognition' in window) { // supportMsg.innerHTML = 'Your browser supports speech recognition.'; // speech recognition API supported // } else { @@ -149,36 +168,39 @@ window.SpeechRecognition = window.webkitSpeechRecognition || window.SpeechRecogn // } const recognition = new window.SpeechRecognition(); -recognition.lang = 'en-US'; +recognition.lang = "en-US"; recognition.interimResults = false; -recordButton.addEventListener('click', (e) => { - e.preventDefault(); - // player.pauseVideo(); - console.log('Record button clicked..Recording....'); - player.pauseVideo(); - recognition.start(); - e.stopPropagation(); +recordButton.addEventListener("click", e => { + e.preventDefault(); + // player.pauseVideo(); + console.log("Record button clicked..Recording...."); + player.pauseVideo(); + recognition.start(); + e.stopPropagation(); }); -recognition.onresult = (event) => { - const speechToText = event.results[0][0].transcript; - if(speechToText.length > 0){ - let questionId = create_UUID(); - let data = { name: roomName, - message: speechToText , - questionId, - QUESTION - }; - - appendQnA(data); - socket.emit('send-chat-message', roomName, speechToText, questionId, name, QUESTION); - console.log({speechToText}); - // console.log('Confidence: ' + event.results[0][0].confidence); - }else{ - console.log({speechToText}); - } -} +recognition.onresult = event => { + const speechToText = event.results[0][0].transcript; + if (speechToText.length > 0) { + let questionId = create_UUID(); + let data = { name: roomName, message: speechToText, questionId, QUESTION }; + + appendQnA(data); + socket.emit( + "send-chat-message", + roomName, + speechToText, + questionId, + name, + QUESTION + ); + console.log({ speechToText }); + // console.log('Confidence: ' + event.results[0][0].confidence); + } else { + console.log({ speechToText }); + } +}; // let supportMsg = document.getElementById('supportMsg'); // //SPeech Synthesis // if ('speechSynthesis' in window) { @@ -189,53 +211,56 @@ recognition.onresult = (event) => { // supportMsg.classList.add('not-supported'); // } -let button = document.getElementById('speak'); +let button = document.getElementById("speak"); function speak(text) { - // Create a new instance of SpeechSynthesisUtterance. - var msg = new SpeechSynthesisUtterance(); - msg.text = text; - msg.volume = 1; - msg.rate = 1; - msg.pitch = 1; - msg.voice = speechSynthesis.getVoices().filter(function(voice) { return voice.name == 'Alex' })[0]; - window.speechSynthesis.speak(msg); - console.log('speech synthesis called:', text ); + // Create a new instance of SpeechSynthesisUtterance. + var msg = new SpeechSynthesisUtterance(); + msg.text = text; + msg.volume = 1; + msg.rate = 1; + msg.pitch = 1; + msg.voice = speechSynthesis.getVoices().filter(function(voice) { + return voice.name == "Alex"; + })[0]; + window.speechSynthesis.speak(msg); + console.log("speech synthesis called:", text); } var done = false; function onPlayerStateChange(event) { - if (event.data == YT.PlayerState.PLAYING && !done) { + if (event.data == YT.PlayerState.PLAYING && !done) { // setTimeout(stopVideo, 6000); done = true; - }else if(event.data == YT.PlayerState.PAUSED){ - console.log({'data': event.target.getCurrentTime()}); - let currentTime = formatTime(event.target.getCurrentTime()); - let message = `Paused the video at ${currentTime} minutes`; - socket.emit('send-chat-message', roomName, message, '', name ,INFO); - } + } else if (event.data == YT.PlayerState.PAUSED) { + console.log({ data: event.target.getCurrentTime() }); + let currentTime = formatTime(event.target.getCurrentTime()); + let message = `Paused the video at ${currentTime} minutes`; + socket.emit("send-chat-message", roomName, message, "", name, INFO); + } } function formatTime(time) { - // console.log(time); - var minutes = Math.floor(time / 60); - var seconds = time - minutes * 60; - seconds = seconds < 10 ? '0' + seconds : seconds; - var s = parseFloat(seconds).toFixed(0); - return minutes + ":" + s; + // console.log(time); + var minutes = Math.floor(time / 60); + var seconds = time - minutes * 60; + seconds = seconds < 10 ? "0" + seconds : seconds; + var s = parseFloat(seconds).toFixed(0); + return minutes + ":" + s; } function stopVideo() { - player.stopVideo(); + player.stopVideo(); } -function create_UUID(){ - var dt = new Date().getTime(); - var uuid = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { - var r = (dt + Math.random()*16)%16 | 0; - dt = Math.floor(dt/16); - return (c=='x' ? r :(r&0x3|0x8)).toString(16); - }); - return uuid; +function create_UUID() { + var dt = new Date().getTime(); + var uuid = "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, function( + c + ) { + var r = (dt + Math.random() * 16) % 16 | 0; + dt = Math.floor(dt / 16); + return (c == "x" ? r : (r & 0x3) | 0x8).toString(16); + }); + return uuid; } - diff --git a/public/javascripts/filterVisdialCaption.js b/public/javascripts/filterVisdialCaption.js new file mode 100644 index 0000000..3d52a74 --- /dev/null +++ b/public/javascripts/filterVisdialCaption.js @@ -0,0 +1,17 @@ +var filterDict = ["","can't tell"]; +[Unit] +Description=worker viscap service %i + +[Service] +Type=simple +User=ubuntu +WorkingDirectory=/home/ubuntu/new/visual-chatbot +Environment="CUDA_VISIBLE_DEVICES=0" +ExecStart=/home/ubuntu/miniconda2/envs/new-vischat/bin/python worker_viscap.py --gpu-ids 0 +ExecReload=/bin/kill -s HUP $MAINPID +ExecStop=/bin/kill -s TERM $MAINPID +Restart=on-abort +PrivateTmp=true + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/public/javascripts/handleCtrlEvents.js b/public/javascripts/handleCtrlEvents.js new file mode 100644 index 0000000..5bc27d5 --- /dev/null +++ b/public/javascripts/handleCtrlEvents.js @@ -0,0 +1,216 @@ +function isKeyPress(event) { + let x = document.getElementById("demo"); + + let asciiValue = event.which; + console.log("ascii value is: ", asciiValue); + switch (asciiValue) { + case 67: + case 99: + x.innerHTML = "The C Key was pressed!"; + console.log("The C Key was pressed!"); + // fetchKeyFrames(); + break; + case 81: + case 113: + x.innerHTML = "The Q Key was pressed!"; + console.log("The Q Key was pressed!"); + askQuestion() + // .then((questionsAsked += 1)) + // .then(console.log(questionsAsked)); + // startChatbot(); + break; + case 69: + case 100: + x.innerHTML = "The D Key was pressed!"; + console.log("The D Key was pressed!"); + fetchImageCaption() + // .then((captionsRequested += 1)) + // .then(console.log(captionsRequested)); + break; + + case 73: + case 105: + x.innerHTML = "The I Key was pressed!"; + if(isTutorial){ + if (player.getPlayerState() == YT.PlayerState.PLAYING) {pauseVideo();} + if (tutorialStarted) { + tutorialSteps[currentInstruction].recurringSpeech(); + } else { + tutorialStarted = true; + setInterval(tutorialProgress, 500); + } + + console.log("The I Key was pressed!"); + } + break; + + case 74: + case 106: + x.innerHTML = "The J Key was pressed!"; + //Add earcon + // if(isVote){ + // updateUserVote(true); + // } + + console.log("The J Key was pressed!"); + break; + + case 75: + case 107: + x.innerHTML = "The K Key was pressed!"; + // if(isVote){ + // updateUserVote(false); + // } + + console.log("The K Key was pressed!"); + break; + + case 82: + case 114: + seekBackward(1); + break; + case 32: + x.innerHTML = "The space Key was pressed!"; + + if (player.getPlayerState() == YT.PlayerState.PLAYING) { + if (isTutorial && tutorialStarted) { + //speak("Video Play"); + playPause += 1; + console.log("This is the file"); + // player.pauseVideo(); + } + pauseVideo();} + else { + if(isTutorial){ + if (tutorialStarted) { + //speak("Video Play"); + player.playVideo(); + } + } + else playVideo(); + } + + break; + } +} +$(window).keypress(function(e) { + if (e.key === " " || e.key === "Spacebar") { + // ' ' is standard, 'Spacebar' was used by IE9 and Firefox < 37 + e.preventDefault(); + console.log("Space pressed"); + if (player.getPlayerState() == YT.PlayerState.PLAYING) pauseVideo(); + else playVideo(); + } +}); + +function isKeyDown(event) { + let x = document.getElementById("demo"); + + if (event.keyCode == "38") { + x.innerHTML = "The Up Arrow Key was pressed!"; + if(isVote){ + updateUserVote(true); + } + } else if (event.keyCode == "40") { + x.innerHTML = "The Down Arrow Key was pressed!"; + if(isVote){ + updateUserVote(false); + } + } + +// if (event.ctrlKey) { +// x.innerHTML = "The CTRL Key was pressed!"; +// } else if (event.keyCode == "37") { +// x.innerHTML = "The Left Arrow Key was pressed!"; +// seekBackward(); +// } else if (event.keyCode == "39") { +// x.innerHTML = "The Right Arrow Key was pressed!"; +// seekForward(); +// } +} + +/** Ctrl + key events variables and functions */ + +let isCtrlPressed = false; + +function isCtrlKeyDown(event) { + let x = document.getElementById("demo"); + + if (isCtrlPressed) { + if (event.which == 81 || event.which == 113) { + x.innerHTML = "The CTRL key + Q Key was pressed!"; + askQuestion(); + } else if (event.which == 67 || event.which == 99) { + x.innerHTML = "The CTRL key + C Key was pressed!"; + fetchKeyFrames(); + } + } else { + if (event.ctrlKey) { + x.innerHTML = "The CTRL key was pressed!"; + isCtrlPressed = true; + } else { + x.innerHTML = "The CTRL key was NOT pressed!"; + isCtrlPressed = false; + } + } +} + +function isCtrlKeyUp(event) { + isCtrlPressed = false; + let x = document.getElementById("demo"); + x.innerHTML = "The key is up now"; +} + +function seekBackward(newTime=5) { + let isStatePlaying = false; + if (player.getPlayerState() == YT.PlayerState.PLAYING) isStatePlaying = true; + + pauseVideo(); + let currentTime = player.getCurrentTime(); + + if (currentTime > 5) player.seekTo(currentTime - newTime); + else player.seekTo(0); + + if (isStatePlaying) playVideo(); +} + +function seekForward() { + let isStatePlaying = false; + if (player.getPlayerState() == YT.PlayerState.PLAYING) isStatePlaying = true; + + pauseVideo(); + let currentTime = player.getCurrentTime(); + // console.log(player.getDuration()); + if (currentTime < player.getDuration() - 5) player.seekTo(currentTime + 5); + else player.seekTo(player.getDuration()); + + if (isStatePlaying) playVideo(); +} + +function updateUserVote(isUpvote){ + + if (isUpvote) + audioVote.src = "../audio/earcons/vote-up.wav"; + else + audioVote.src = "../audio/earcons/vote-down2.wav"; + + let currentTime = -1; + if (player){ + currentTime = player.getCurrentTime(); + currentTime = currentTime.toFixed(2); + } + + let data = { + userId: volunteerId, + videoId: youtubeId, + isUpvote, + youtubeTimestamp: currentTime, + conditionType + }; + + $.post("/userStudy4/addVote", data, function(res) { + console.log("Vote Added"); + console.log(res); + }); + +} \ No newline at end of file diff --git a/public/javascripts/index.js b/public/javascripts/index.js index f3357cf..4b7e1ac 100644 --- a/public/javascripts/index.js +++ b/public/javascripts/index.js @@ -1,5 +1,5 @@ let player; -let youtubeId = ''; +let youtubeId = ""; let videotime = 0; let timeupdater = null; let sceneIdToListPositionMap = new Map(); @@ -7,455 +7,504 @@ let sceneToBlockMap = new Map(); let userId; let prevSceneNum = -1; let timeAtWhichLastPaused = -1; -let previouslyHighlightedContainerId = '#doesnotexist'; +let previouslyHighlightedContainerId = "#doesnotexist"; let enableAi = false; let autoPauseAtEachScene = true; let timeElapsedOnPage = 0; let isTimerPaused = false; $(document).ready(function() { - $('#btn-video-breakpoint-toggle').prop("checked", true); - userId = getUrlVars()["u"]; - enableAi = (getUrlVars()["ai"] == 'yes') ? true : false; - let href = location.href; - const lastPathSegment = new URL(href).pathname.split('/').pop(); - youtubeId = lastPathSegment; - fetchVideoInfo(youtubeId); - - $("#btnScroll").click(function() { - resetVidModifiedTableData(); - }); - $('#btnSubmit').click(async function() { - let shoudShowAlert = true; - updateAll(shoudShowAlert); - }); - $('#btnTemp').click(async function() { - updateMainTables(); - }); - $('#btn-video-breakpoint-toggle').change(function() { - if($(this).is(":checked")) { - autoPauseAtEachScene = true; - } else { - autoPauseAtEachScene = false; - } - }); - $("#btn-play-pause").click(function() { - onPlayPauseButtonClick(); - }); - $('#btn-timer').click(function () { - isTimerPaused = !isTimerPaused; - if(isTimerPaused) { - $('#btn-timer').attr('src', '/images/play_white.png'); - } else { - $('#btn-timer').attr('src', '/images/pause_white.png'); - } - }); - setInterval(incrementSeconds, 1000); + $("#btn-video-breakpoint-toggle").prop("checked", true); + userId = getUrlVars()["u"]; + enableAi = getUrlVars()["ai"] == "yes" ? true : false; + let href = location.href; + const lastPathSegment = new URL(href).pathname.split("/").pop(); + youtubeId = lastPathSegment; + fetchVideoInfo(youtubeId); + + $("#btnScroll").click(function() { + resetVidModifiedTableData(); + }); + $("#btnSubmit").click(async function() { + let shoudShowAlert = true; + updateAll(shoudShowAlert); + }); + $("#btnTemp").click(async function() { + updateMainTables(); + }); + $("#btn-video-breakpoint-toggle").change(function() { + if ($(this).is(":checked")) { + autoPauseAtEachScene = true; + } else { + autoPauseAtEachScene = false; + } + }); + $("#btn-play-pause").click(function() { + onPlayPauseButtonClick(); + }); + $("#btn-timer").click(function() { + isTimerPaused = !isTimerPaused; + if (isTimerPaused) { + $("#btn-timer").attr("src", "/images/play_white.png"); + } else { + $("#btn-timer").attr("src", "/images/pause_white.png"); + } + }); + setInterval(incrementSeconds, 1000); }); function incrementSeconds() { - if(!isTimerPaused) { - timeElapsedOnPage += 1; - $('#txt-timeElaspsed').html(formatTimeForTimer(timeElapsedOnPage)); - } + if (!isTimerPaused) { + timeElapsedOnPage += 1; + $("#txt-timeElaspsed").html(formatTimeForTimer(timeElapsedOnPage)); + } } - function fetchVideoInfo(youtubeId) { - console.log(userId); - // let targetUrl = '/fetchVideoData' + '?videoid=' + youtubeId; - let targetUrl = `/fetchVideoData?videoid=${youtubeId}&userId=${userId}` - $.get(targetUrl, function (response) { - let vidJson = response; - let vidTitle = vidJson.title; - let vidLength = vidJson.duration; - let blocksJsonArr = vidJson.info; - overrideData = vidJson.overrideData || []; - overrideData = overrideData.map( (over) => { - let { deletedScene='[]', ...rest } = over - return { - deletedScene: JSON.parse(deletedScene), - ...rest, - } - }) - $('#txtVidtitle').html(vidTitle); - - //setting the data attribute for list-container\ - let listContainer = document.querySelector('.list-container'); - if(overrideData.length > 0){ - listContainer.dataset.sceneDetails = JSON.stringify(overrideData); - }else{ - let sceneArr = []; - for(let i=0; i`; - let getQuestionBtn = ``; - let qaContainer = `
    `; - let createMoreQues = ``; - let mergeScene = ``; - let newQuestion = `${getQuestionBtn}
    ${qaContainer}
    ${createMoreQues}
    ${doneSceneBtn}
    ${mergeScene} ${mergeMessage}
    `; - - - let outputHtml = `
  • ` - + `` - + '' - + '' - + '
    Scene ' + scene_num + `` + scStartTimePretty + ' to ' + scFinishTimePretty + ' minutes
    Description
    ' - + '
    Text On-Screen' - + '
    ' - + '
    ' - + `
    ` - + newQuestion; - - return outputHtml; -} - - - -const renderQnaList = (qnaList) => { - qnaList.forEach(qa => { - let { answerId, sceneId, questionId, question, answer } = qa; - let currentContainer = `qna_container:${sceneId}` - let cont = document.getElementById(currentContainer); - const qaElement = getNewQnA(question, answer, questionId, answerId); - qaElement.dataset.jsonData = JSON.stringify(qa); - cont.appendChild(qaElement); - }); + console.log({ currScene }); + let sceneId = currScene.sc_id; + let sceneNum = currScene.sc_num; + let scStartTime = currScene.start_time; + let scFinishTime = currScene.finish_time; + let sceneOcr = enableAi ? $.trim(currScene.original_ocr) : ""; + let sceneDesc = enableAi ? $.trim(currScene.original_description) : ""; + let scStartTimePretty = millisToMinutesAndSeconds(scStartTime); + let scFinishTimePretty = millisToMinutesAndSeconds(scFinishTime); + let mergeMessage = + "Please merge the scene before generating questions for the next scene"; + let doneSceneBtn = ``; + let getQuestionBtn = ``; + let qaContainer = `
    `; + let createMoreQues = ``; + let mergeScene = ``; + let newQuestion = `${getQuestionBtn}
    ${qaContainer}
    ${createMoreQues}
    ${doneSceneBtn}
    ${mergeScene} ${mergeMessage}
    `; + + let outputHtml = + `
  • ` + + `` + + "' + + '" + + "
    Scene " + + scene_num + + `` + + scStartTimePretty + + " to " + + scFinishTimePretty + + ' minutes
    Description
    ' + + '
    Text On-Screen" + + '
    ' + + '
    " + + `
    ` + + newQuestion; + + return outputHtml; } +const renderQnaList = qnaList => { + qnaList.forEach(qa => { + let { answerId, sceneId, questionId, question, answer } = qa; + let currentContainer = `qna_container:${sceneId}`; + let cont = document.getElementById(currentContainer); + const qaElement = getNewQnA(question, answer, questionId, answerId); + qaElement.dataset.jsonData = JSON.stringify(qa); + cont.appendChild(qaElement); + }); +}; /* created by Vaishali create Question answer on click of button "addQuestion" */ -function getNewQnA (ques, ans, quesId, ansId) { - let qa = document.createElement('div'); +function getNewQnA(ques, ans, quesId, ansId) { + let qa = document.createElement("div"); - let del = `` - qa.innerHTML = `
    + let del = ``; + qa.innerHTML = `
    - + - + ${del}
    `; - return qa; -}; + return qa; +} /* created by Vaishali adding eventListener on add question btn click, get question click */ -document.getElementById('ul-scene-list').addEventListener('click', (e) => { - console.log('click event TARGET: ', e.target); - - if(e.target.id.includes('create_question:')) { - let index = e.target.id.indexOf(':')+1; - let textDescArea = e.target.id.substring(index); - document.getElementById(`qna_container:${textDescArea}`).appendChild(getNewQnA()); - }else if (e.target.id.includes('get_question:')) { - let index = e.target.id.indexOf(':')+1; - let textDescArea = e.target.id.substring(index); - let descId = 's_d_'+ textDescArea; - let desc = document.getElementById(descId).value; - - console.log('descId: ', descId, 'val: ', desc); - let rows = desc.split("\n"); - for (var i = 0; i < rows.length; i++) { - console.log('ROWS:', rows[i]); +document.getElementById("ul-scene-list").addEventListener("click", e => { + console.log("click event TARGET: ", e.target); + + if (e.target.id.includes("create_question:")) { + let index = e.target.id.indexOf(":") + 1; + let textDescArea = e.target.id.substring(index); + document + .getElementById(`qna_container:${textDescArea}`) + .appendChild(getNewQnA()); + } else if (e.target.id.includes("get_question:")) { + let index = e.target.id.indexOf(":") + 1; + let textDescArea = e.target.id.substring(index); + let descId = "s_d_" + textDescArea; + let desc = document.getElementById(descId).value; + + console.log("descId: ", descId, "val: ", desc); + let rows = desc.split("\n"); + for (var i = 0; i < rows.length; i++) { + console.log("ROWS:", rows[i]); + } + let postJson = { + sentence: rows + }; + + let questionArr = []; + $.post("/us3/question/generate", postJson, function(res) { + rows = res.result; + for (let i = 0; i < rows.length; i++) { + if (rows[i].response.status === "success") { + console.log({ QUESTION: rows[i].response.question }); + questionArr.push(rows[i]); + let response = rows[i].response; + let qnAelement = getNewQnA(response.question, response.answer); + // e.target.nextSibling.nextSibling.appendChild(qnAelement); + + document + .getElementById(`qna_container:${textDescArea}`) + .appendChild(qnAelement); } - let postJson = { - "sentence" : rows - }; - - let questionArr = []; - $.post('/us3/question/generate', postJson, function (res) { - rows = res.result; - for(let i=0; i< rows.length ; i++){ - if(rows[i].response.status === 'success'){ - console.log({QUESTION: rows[i].response.question}); - questionArr.push(rows[i]); - let response = rows[i].response; - let qnAelement = getNewQnA(response.question, response.answer); - // e.target.nextSibling.nextSibling.appendChild(qnAelement); - - document.getElementById(`qna_container:${textDescArea}`).appendChild(qnAelement) - - } - } - console.log('RESULT ', res.result); - console.log({questionArr}); - }); - }else if(e.target.id.includes('delBtn')){ - console.log('INSIDE DELETEEEE ', e.target); - let [txt, questionId, answerId] = e.target.id.split(':') - let jsonBody = { - "questionId" : questionId, - "answerId" : answerId - }; - console.log('DELETEJSON ', jsonBody); - $.ajax({ - type: 'DELETE', - url : '/us3/question/deleteQuestion', - dataType: "json", - async:true, - data: jsonBody, - success: function (response) { - console.log('SUCCESS:', response); - }, - error: function (response, textStatus, errorThrown) { - console.log(errorThrown); - } - }); - } else if(e.target.id === 'doneSceneBtn') { - - playVideo(); - - } else if(e.target.id.includes('merge_scene')){ - let listContainer = document.querySelector('.list-container'); - sceneDetails = JSON.parse(listContainer.dataset.sceneDetails); - - let result = confirm("Do you want to merge current scene with the next?"); - if (result) { - //Logic to delete the item - console.log('inside merge scene'); - let index = e.target.id.indexOf(':')+1; - let curSceneId = e.target.id.substring(index); - console.log({curSceneId}); - let curSceneDescElement = document.getElementById('s_d_'+ curSceneId); - let curSceneOCRElement = document.getElementById('s_o_'+ curSceneId); - console.log({curSceneDescElement}); - - //next Sibling - let nextSibling = document.getElementById(e.target.id).parentElement.parentElement.nextSibling; - let nextSiblingSceneId = nextSibling.children[0].id; - let indexNextSib = nextSiblingSceneId.indexOf(':')+1; - nextSiblingSceneId = nextSiblingSceneId.substring(indexNextSib); - let nextSceneData = sceneDetails.filter( el => el.sc_id === nextSiblingSceneId )[0] - let currSceneData = sceneDetails.filter( el => el.sc_id === curSceneId )[0] - - if(!currSceneData.deletedScene){ - let deletedSceneArr = []; - deletedSceneArr.push(nextSiblingSceneId); - currSceneData.deletedScene = deletedSceneArr; - }else{ - if(nextSceneData.deletedScene){ - currSceneData.deletedScene.push(...nextSceneData.deletedScene); - } - - currSceneData.deletedScene.push(nextSiblingSceneId); - } - - console.log({ nextSceneData, currSceneData}) - - let newStartTime = Math.min(currSceneData.start_time, nextSceneData.start_time); - let newEndTime = Math.max(currSceneData.finish_time, nextSceneData.finish_time); - currSceneData.finish_time = newEndTime; - currSceneData.start_time = newStartTime; - console.log('New time', newStartTime, '->', newEndTime); - - document.getElementById(`time:${curSceneId}`).innerHTML = `${millisToMinutesAndSeconds(newStartTime)} to ${millisToMinutesAndSeconds(newEndTime)} minutes` - // console.log({allSceneArr}); - console.log({nextSiblingSceneId}); - let nextSceneDescElement = document.getElementById('s_d_'+ nextSiblingSceneId); - let nextSceneOCRElement = document.getElementById('s_o_'+ nextSiblingSceneId); - console.log({nextSceneDescElement}); - - curSceneDescElement.value += '\n' + nextSceneDescElement.value; - curSceneOCRElement.value += '\n' + nextSceneOCRElement.value; - // document.getElementById(curSceneDescElement+'.'+'txt-scene-description').autogrow(); - nextSibling.remove(); - //delete next scene - let jsonBody = { - "sceneId" : nextSiblingSceneId - }; - - console.log(e.target.id); - console.log({"Inside merge:" : sceneDetails}); - listContainer.dataset.sceneDetails = JSON.stringify(sceneDetails); - updateAll(false); - window.location.reload(); - // fetchVideoInfo(youtubeId); - // let currentTime = player.getCurrentTime(); - // player.seekTo(currentTime); - + } + console.log("RESULT ", res.result); + console.log({ questionArr }); + }); + } else if (e.target.id.includes("delBtn")) { + console.log("INSIDE DELETEEEE ", e.target); + let [txt, questionId, answerId] = e.target.id.split(":"); + let jsonBody = { + questionId: questionId, + answerId: answerId + }; + console.log("DELETEJSON ", jsonBody); + $.ajax({ + type: "DELETE", + url: "/us3/question/deleteQuestion", + dataType: "json", + async: true, + data: jsonBody, + success: function(response) { + console.log("SUCCESS:", response); + }, + error: function(response, textStatus, errorThrown) { + console.log(errorThrown); + } + }); + } else if (e.target.id === "doneSceneBtn") { + playVideo(); + } else if (e.target.id.includes("merge_scene")) { + let listContainer = document.querySelector(".list-container"); + sceneDetails = JSON.parse(listContainer.dataset.sceneDetails); + + let result = confirm("Do you want to merge current scene with the next?"); + if (result) { + //Logic to delete the item + console.log("inside merge scene"); + let index = e.target.id.indexOf(":") + 1; + let curSceneId = e.target.id.substring(index); + console.log({ curSceneId }); + let curSceneDescElement = document.getElementById("s_d_" + curSceneId); + let curSceneOCRElement = document.getElementById("s_o_" + curSceneId); + console.log({ curSceneDescElement }); + + //next Sibling + let nextSibling = document.getElementById(e.target.id).parentElement + .parentElement.nextSibling; + let nextSiblingSceneId = nextSibling.children[0].id; + let indexNextSib = nextSiblingSceneId.indexOf(":") + 1; + nextSiblingSceneId = nextSiblingSceneId.substring(indexNextSib); + let nextSceneData = sceneDetails.filter( + el => el.sc_id === nextSiblingSceneId + )[0]; + let currSceneData = sceneDetails.filter(el => el.sc_id === curSceneId)[0]; + + if (!currSceneData.deletedScene) { + let deletedSceneArr = []; + deletedSceneArr.push(nextSiblingSceneId); + currSceneData.deletedScene = deletedSceneArr; + } else { + if (nextSceneData.deletedScene) { + currSceneData.deletedScene.push(...nextSceneData.deletedScene); } + currSceneData.deletedScene.push(nextSiblingSceneId); + } + + console.log({ nextSceneData, currSceneData }); + + let newStartTime = Math.min( + currSceneData.start_time, + nextSceneData.start_time + ); + let newEndTime = Math.max( + currSceneData.finish_time, + nextSceneData.finish_time + ); + currSceneData.finish_time = newEndTime; + currSceneData.start_time = newStartTime; + console.log("New time", newStartTime, "->", newEndTime); + + document.getElementById( + `time:${curSceneId}` + ).innerHTML = `${millisToMinutesAndSeconds( + newStartTime + )} to ${millisToMinutesAndSeconds(newEndTime)} minutes`; + // console.log({allSceneArr}); + console.log({ nextSiblingSceneId }); + let nextSceneDescElement = document.getElementById( + "s_d_" + nextSiblingSceneId + ); + let nextSceneOCRElement = document.getElementById( + "s_o_" + nextSiblingSceneId + ); + console.log({ nextSceneDescElement }); + + curSceneDescElement.value += "\n" + nextSceneDescElement.value; + curSceneOCRElement.value += "\n" + nextSceneOCRElement.value; + // document.getElementById(curSceneDescElement+'.'+'txt-scene-description').autogrow(); + nextSibling.remove(); + //delete next scene + let jsonBody = { + sceneId: nextSiblingSceneId + }; + + console.log(e.target.id); + console.log({ "Inside merge:": sceneDetails }); + listContainer.dataset.sceneDetails = JSON.stringify(sceneDetails); + updateAll(false) + .then(() => { + fetchVideoInfo(youtubeId); + }) + .catch(e => { + alert("Error merging scenes"); + }); } + } }); - function millisToMinutesAndSeconds(secs) { - var minutes = Math.floor(secs / 60); - var seconds = ((secs % 60) / 1).toFixed(1); - return minutes + ":" + seconds; -} + var minutes = Math.floor(secs / 60); + var seconds = ((secs % 60) / 1).toFixed(1); + return minutes + ":" + seconds; +} function scrollToPosition(position) { - position--; - $('#ul-scene-list').animate({ - scrollTop: $('#ul-scene-list>.newscene-list:eq(' + position + ')').position().top - - $('#ul-scene-list>.newscene-list:eq(0)').position().top - }, 'slow'); - let newSceneList = document.querySelectorAll('.newscene-list'); - - newSceneList.forEach(sceneItem => { - sceneItem.firstChild.classList.remove('highlight'); - }) - - newSceneList[position].firstChild.classList.add('highlight'); - - // if(position > 0) { - // newSceneList[position-1].firstChild.classList.remove('highlight'); - // } - console.log({position}); - console.log({previouslyHighlightedContainerId}); - previouslyHighlightedContainerId = '#ul-scene-list>.newscene-list:eq(' + position + ')' + ' .scene-container'; + position--; + $("#ul-scene-list").animate( + { + scrollTop: + $("#ul-scene-list>.newscene-list:eq(" + position + ")").position().top - + $("#ul-scene-list>.newscene-list:eq(0)").position().top + }, + "slow" + ); + let newSceneList = document.querySelectorAll(".newscene-list"); + + newSceneList.forEach(sceneItem => { + sceneItem.firstChild.classList.remove("highlight"); + }); + + newSceneList[position].firstChild.classList.add("highlight"); + + // if(position > 0) { + // newSceneList[position-1].firstChild.classList.remove('highlight'); + // } + console.log({ position }); + console.log({ previouslyHighlightedContainerId }); + previouslyHighlightedContainerId = + "#ul-scene-list>.newscene-list:eq(" + position + ")" + " .scene-container"; } function onYouTubeIframeAPIReady() { - player = new YT.Player('player', { - height: '270', - width: '480', - videoId: youtubeId, - playerVars: { - autoplay: 1, - enablejsapi: 1, - cc_load_policy: 1, - controls: 1, - fs: 0, - iv_load_policy: 3, - modestbranding: 1, - rel: 0, - showinfo: 0, - wmode: 'opaque' - }, - events: { - 'onReady': onPlayerReady, - 'onStateChange': onPlayerStateChange - } - }); + player = new YT.Player("player", { + height: "270", + width: "480", + videoId: youtubeId, + playerVars: { + autoplay: 1, + enablejsapi: 1, + cc_load_policy: 1, + controls: 1, + fs: 0, + iv_load_policy: 3, + modestbranding: 1, + rel: 0, + showinfo: 0, + wmode: "opaque" + }, + events: { + onReady: onPlayerReady, + onStateChange: onPlayerStateChange + } + }); } function onPlayerReady(event) { - $('#total_time').text(formatTime(player.getDuration())); - timeupdater = setInterval(updateTime, 100); + $("#total_time").text(formatTime(player.getDuration())); + timeupdater = setInterval(updateTime, 100); + document.getElementById("player").tabIndex = -1; } function updateTime() { - var oldTime = videotime; - if(player && player.getCurrentTime) { - videotime = player.getCurrentTime(); - } - if(videotime !== oldTime) { - onProgress(videotime); - } - $('#elapsed_time').text(formatTime(player.getCurrentTime())); + var oldTime = videotime; + if (player && player.getCurrentTime) { + videotime = player.getCurrentTime(); + } + if (videotime !== oldTime) { + onProgress(videotime); + } + $("#elapsed_time").text(formatTime(player.getCurrentTime())); } // when the time changes, this will be called. function onProgress(currentTime) { - currentTime = currentTime.toFixed(3); - let currScene = getSceneFromTime(currentTime); - let sceneId = currScene.sc_id; - let sceneNum = currScene.sc_num; - let sceneEndTime = currScene.finish_time; - if(prevSceneNum != sceneNum) { - let listContainer = document.querySelector('.list-container'); - let sceneDetails = JSON.parse(listContainer.dataset.sceneDetails); - let sceneIdToListPositionMap = populateSceneIdToListPositionMap(sceneDetails); - let listPosition = sceneIdToListPositionMap.get(sceneId); - console.log({sceneIdToListPositionMap, listPosition}); - scrollToPosition(listPosition); - } - prevSceneNum = sceneNum; - if(timeAtWhichLastPaused > currentTime) { - timeAtWhichLastPaused = currentTime; - } - if(sceneEndTime - currentTime < 0.1 && currentTime - timeAtWhichLastPaused > 0.5 && autoPauseAtEachScene) { - pauseVideo(); - timeAtWhichLastPaused = currentTime; - } - let sceneElementId = '#sc_cont_' + sceneId; - // $(previouslyHighlightedContainerId).removeClass('highlight'); - $(sceneElementId).addClass('highlight'); - previouslyHighlightedContainerId = sceneElementId; + currentTime = currentTime.toFixed(3); + let currScene = getSceneFromTime(currentTime); + let sceneId = currScene.sc_id; + let sceneNum = currScene.sc_num; + let sceneEndTime = currScene.finish_time; + if (prevSceneNum != sceneNum) { + let listContainer = document.querySelector(".list-container"); + let sceneDetails = JSON.parse(listContainer.dataset.sceneDetails); + let sceneIdToListPositionMap = populateSceneIdToListPositionMap( + sceneDetails + ); + let listPosition = sceneIdToListPositionMap.get(sceneId); + console.log({ sceneIdToListPositionMap, listPosition }); + scrollToPosition(listPosition); + } + prevSceneNum = sceneNum; + if (timeAtWhichLastPaused > currentTime) { + timeAtWhichLastPaused = currentTime; + } + if ( + sceneEndTime - currentTime < 0.1 && + currentTime - timeAtWhichLastPaused > 0.5 && + autoPauseAtEachScene + ) { + pauseVideo(); + timeAtWhichLastPaused = currentTime; + } + let sceneElementId = "#sc_cont_" + sceneId; + // $(previouslyHighlightedContainerId).removeClass('highlight'); + $(sceneElementId).addClass("highlight"); + previouslyHighlightedContainerId = sceneElementId; } function onPlayerStateChange(event) { - let playerState = event.data; - switch(playerState) { - case YT.PlayerState.PLAYING: - console.log('i was called: playing state') - $("#btn-play-pause").html(''); - break; - case YT.PlayerState.PAUSED: - $("#btn-play-pause").html(''); - timeAtWhichLastPaused = player.getCurrentTime(); - console.log(timeAtWhichLastPaused); - console.log('i was called: paused state') - break; - case YT.PlayerState.UNSTARTED: - case YT.PlayerState.ENDED: - case YT.PlayerState.CUED:n - $("#btn-play-pause").html(''); - timeAtWhichLastPaused = player.getCurrentTime(); - console.log('i was called: cued state') - break; - } + let playerState = event.data; + switch (playerState) { + case YT.PlayerState.PLAYING: + console.log("i was called: playing state"); + $("#btn-play-pause").html( + '' + ); + break; + case YT.PlayerState.PAUSED: + $("#btn-play-pause").html( + '' + ); + timeAtWhichLastPaused = player.getCurrentTime(); + console.log(timeAtWhichLastPaused); + console.log("i was called: paused state"); + break; + case YT.PlayerState.UNSTARTED: + case YT.PlayerState.ENDED: + case YT.PlayerState.CUED: + $("#btn-play-pause").html( + '' + ); + timeAtWhichLastPaused = player.getCurrentTime(); + console.log("i was called: cued state"); + break; + } } function playVideo() { - player.playVideo(); + player.playVideo(); } function pauseVideo() { - player.pauseVideo(); + player.pauseVideo(); } function stopVideo() { - player.stopVideo(); + player.stopVideo(); } /* @@ -463,250 +512,261 @@ created by Vaishali iterating over all QnAcontainers, saving entire info for the ques and answers that already exists in the DB, sending new qnAs */ -const getNewQnAData = () =>{ - const qnaData = {} - var allQAContainer = $('div[id^="qna_container:"]'); - - for(let i=0; i< allQAContainer.length; i++) { - const qaList = allQAContainer[i].children; - for(let childIndex = 0; childIndex< qaList.length; childIndex++){ - console.log('id of parent', allQAContainer[i].id.substring(14)); - let sceneId = allQAContainer[i].id.substring(14) - const currentElement = qaList[childIndex]; - var question = currentElement.children[0].children[1].value; - var answer = currentElement.children[0].children[3].value; - console.log({question, answer}) - - qnaData[sceneId] = qnaData[sceneId] || []; - if(currentElement.dataset.jsonData) { - const tempData = JSON.parse(currentElement.dataset.jsonData); //saving coming json data for the exisiting qnA - qnaData[sceneId].push({question, answer, answerId: tempData.answerId, questionId: tempData.questionId }); - } else { - qnaData[sceneId].push({videoId:youtubeId, question, answer })//pushing only questio and answer value to the exisiting qnA. - } - } +const getNewQnAData = () => { + const qnaData = {}; + var allQAContainer = $('div[id^="qna_container:"]'); + + for (let i = 0; i < allQAContainer.length; i++) { + const qaList = allQAContainer[i].children; + for (let childIndex = 0; childIndex < qaList.length; childIndex++) { + console.log("id of parent", allQAContainer[i].id.substring(14)); + let sceneId = allQAContainer[i].id.substring(14); + const currentElement = qaList[childIndex]; + var question = currentElement.children[0].children[1].value; + var answer = currentElement.children[0].children[3].value; + console.log({ question, answer }); + + qnaData[sceneId] = qnaData[sceneId] || []; + if (currentElement.dataset.jsonData) { + const tempData = JSON.parse(currentElement.dataset.jsonData); //saving coming json data for the exisiting qnA + qnaData[sceneId].push({ + question, + answer, + answerId: tempData.answerId, + questionId: tempData.questionId + }); + } else { + qnaData[sceneId].push({ videoId: youtubeId, question, answer }); //pushing only questio and answer value to the exisiting qnA. + } } - return qnaData; -} + } + return qnaData; +}; async function updateAll(shoudShowAlert) { - let postJson = fetchJsonBodyForAllTextFields(userId); - const qnaData = getNewQnAData(); - postJson.qnaData = qnaData; - postJson.videoId = youtubeId; - console.log({'HERE': postJson}); - try{ - await $.post('/saveAiDescription', postJson, function (response) { - if(shoudShowAlert) { - if(response.success) { - alert('Submitted Successfully\nYou will be redirected to the Video List Page'); - window.history.back(); - } else { - console.log(response); - alert('Update Status: ' + response.success); - } - } - }); - }catch(e){ - - console.log(e); - // response.send(e); - } - + let postJson = fetchJsonBodyForAllTextFields(userId); + const qnaData = getNewQnAData(); + postJson.qnaData = qnaData; + postJson.videoId = youtubeId; + console.log({ HERE: postJson }); + try { + await $.post("/saveAiDescription", postJson, function(response) { + if (shoudShowAlert) { + if (response.success) { + alert( + "Submitted Successfully\nYou will be redirected to the Video List Page" + ); + window.history.back(); + } else { + console.log(response); + alert("Update Status: " + response.success); + } + } + console.log(response); + }); + } catch (e) { + console.log(e); + // response.send(e); + } } //added start time, end time and has Ai for saving in descrption table function fetchJsonBodyForAllTextFields(userId) { - let listContainer = document.querySelector('.list-container'); - sceneDetails = JSON.parse(listContainer.dataset.sceneDetails); - - let parentJsonBody = {}; - let sceneArrJson = []; - - console.log({'INSIDE fetchJsonBodyForAllTextFields ' : sceneDetails}); - let currScene ; - for(let i=0; i < sceneDetails.length; i++) { - currScene = sceneDetails[i]; - let currSceneId = currScene.sc_id; - let currSceneDescTextId = '#s_d_' + currSceneId; - let currSceneOcrTextId = '#s_o_' + currSceneId; - let sceneJsonObj = {}; - sceneJsonObj.scene_id = currSceneId; - let csTestbox = $(currSceneDescTextId); - // if(csTestbox && csTestbox.val()){ - // sceneJsonObj.modified_description = csTestbox.val().trim(); - // sceneJsonObj.modified_ocr = $(currSceneOcrTextId).val().trim(); - // sceneJsonObj.start_time = currScene.start_time; - // sceneJsonObj.end_time = currScene.finish_time; - // sceneJsonObj.has_ai = enableAi; - // sceneJsonObj.deletedScene = currScene.deletedScene; - // sceneArrJson.push(sceneJsonObj); - // }else{ - - if(csTestbox && csTestbox.val()){ - sceneJsonObj.modified_description = csTestbox.val().trim(); - }else{ - sceneJsonObj.modified_description = ''; - } - - let ocrValue = $(currSceneOcrTextId); - if( ocrValue && ocrValue.val()){ - sceneJsonObj.modified_ocr = ocrValue.val().trim(); - }else{ - sceneJsonObj.modified_ocr = ''; - } - sceneJsonObj.start_time = currScene.start_time; - sceneJsonObj.end_time = currScene.finish_time; - sceneJsonObj.has_ai = enableAi; - sceneJsonObj.deletedScene = currScene.deletedScene; - sceneArrJson.push(sceneJsonObj); + let listContainer = document.querySelector(".list-container"); + sceneDetails = JSON.parse(listContainer.dataset.sceneDetails); + + let parentJsonBody = {}; + let sceneArrJson = []; + + console.log({ "INSIDE fetchJsonBodyForAllTextFields ": sceneDetails }); + let currScene; + for (let i = 0; i < sceneDetails.length; i++) { + currScene = sceneDetails[i]; + let currSceneId = currScene.sc_id; + let currSceneDescTextId = "#s_d_" + currSceneId; + let currSceneOcrTextId = "#s_o_" + currSceneId; + let sceneJsonObj = {}; + sceneJsonObj.scene_id = currSceneId; + let csTestbox = $(currSceneDescTextId); + // if(csTestbox && csTestbox.val()){ + // sceneJsonObj.modified_description = csTestbox.val().trim(); + // sceneJsonObj.modified_ocr = $(currSceneOcrTextId).val().trim(); + // sceneJsonObj.start_time = currScene.start_time; + // sceneJsonObj.end_time = currScene.finish_time; + // sceneJsonObj.has_ai = enableAi; + // sceneJsonObj.deletedScene = currScene.deletedScene; + // sceneArrJson.push(sceneJsonObj); + // }else{ + + if (csTestbox && csTestbox.val()) { + sceneJsonObj.modified_description = csTestbox.val().trim(); + } else { + sceneJsonObj.modified_description = ""; } - - - parentJsonBody.user_id = userId; - parentJsonBody.scene_arr = sceneArrJson; - parentJsonBody.time_taken = timeElapsedOnPage; - console.log({parentJsonBody}); - return parentJsonBody; + let ocrValue = $(currSceneOcrTextId); + if (ocrValue && ocrValue.val()) { + sceneJsonObj.modified_ocr = ocrValue.val().trim(); + } else { + sceneJsonObj.modified_ocr = ""; + } + sceneJsonObj.start_time = currScene.start_time; + sceneJsonObj.end_time = currScene.finish_time; + sceneJsonObj.has_ai = enableAi; + sceneJsonObj.deletedScene = currScene.deletedScene; + sceneArrJson.push(sceneJsonObj); + } + + parentJsonBody.user_id = userId; + parentJsonBody.scene_arr = sceneArrJson; + parentJsonBody.time_taken = timeElapsedOnPage; + + console.log({ parentJsonBody }); + return parentJsonBody; } async function updateMainTables() { - let postJson = fetchJsonBodyForAllTextFields2(allSceneArr, allBlockArr); - - console.log({postJson}) - await $.post('/updateBlockAndScene', postJson, function (response) { - if(response.success) { - window.alert('Updated Successfully'); - } else { - window.alert('Update Failed'); - } - }); + let postJson = fetchJsonBodyForAllTextFields2(allSceneArr, allBlockArr); + + console.log({ postJson }); + await $.post("/updateBlockAndScene", postJson, function(response) { + if (response.success) { + window.alert("Updated Successfully"); + } else { + window.alert("Update Failed"); + } + }); } function fetchJsonBodyForAllTextFields2(allSceneArr, allBlockArr, userId) { - let parentJsonBody = {}; - let sceneArrJson = []; - let blockArrJson = []; - for(let i=0; i`; + let getQuestionBtn = ``; + let qaContainer = `
    `; + let createMoreQues = ``; + let mergeScene = ``; + let newQuestion = `${getQuestionBtn}
    ${qaContainer}
    ${createMoreQues}
    ${doneSceneBtn}
    ${mergeScene} ${mergeMessage}
    `; + + let outputHtml = + `
  • ` + + `` + + "' + + '" + + "
    Scene " + + scene_num + + `` + + scStartTimePretty + + " to " + + scFinishTimePretty + + ' minutes
    Description
    ' + + '
    Text On-Screen" + + '
    ' + + '
    " + + `
    ` + + newQuestion; + + return outputHtml; +} + +const renderQnaList = qnaList => { + qnaList.forEach(qa => { + let { answerId, sceneId, questionId, question, answer } = qa; + let currentContainer = `qna_container:${sceneId}`; + let cont = document.getElementById(currentContainer); + const qaElement = getNewQnA(question, answer, questionId, answerId); + qaElement.dataset.jsonData = JSON.stringify(qa); + cont.appendChild(qaElement); + }); +}; + +/* +created by Vaishali +create Question answer on click of button "addQuestion" +*/ +function getNewQnA(ques, ans, quesId, ansId) { + let qa = document.createElement("div"); + + let del = ``; + qa.innerHTML = `
    + + + + + ${del} +
    `; + return qa; +} + +/* +created by Vaishali +adding eventListener on add question btn click, get question click +*/ +document.getElementById("ul-scene-list").addEventListener("click", e => { + console.log("click event TARGET: ", e.target); + + if (e.target.id.includes("create_question:")) { + let index = e.target.id.indexOf(":") + 1; + let textDescArea = e.target.id.substring(index); + document + .getElementById(`qna_container:${textDescArea}`) + .appendChild(getNewQnA()); + } else if (e.target.id.includes("get_question:")) { + let index = e.target.id.indexOf(":") + 1; + let textDescArea = e.target.id.substring(index); + let descId = "s_d_" + textDescArea; + let desc = document.getElementById(descId).value; + + console.log("descId: ", descId, "val: ", desc); + let rows = desc.split("\n"); + for (var i = 0; i < rows.length; i++) { + console.log("ROWS:", rows[i]); + } + let postJson = { + sentence: rows + }; + + let questionArr = []; + $.post("/us3/question/generate", postJson, function(res) { + rows = res.result; + for (let i = 0; i < rows.length; i++) { + if (rows[i].response.status === "success") { + console.log({ QUESTION: rows[i].response.question }); + questionArr.push(rows[i]); + let response = rows[i].response; + let qnAelement = getNewQnA(response.question, response.answer); + // e.target.nextSibling.nextSibling.appendChild(qnAelement); + + document + .getElementById(`qna_container:${textDescArea}`) + .appendChild(qnAelement); + } + } + console.log("RESULT ", res.result); + console.log({ questionArr }); + }); + } else if (e.target.id.includes("delBtn")) { + console.log("INSIDE DELETEEEE ", e.target); + let [txt, questionId, answerId] = e.target.id.split(":"); + let jsonBody = { + questionId: questionId, + answerId: answerId + }; + console.log("DELETEJSON ", jsonBody); + $.ajax({ + type: "DELETE", + url: "/us3/question/deleteQuestion", + dataType: "json", + async: true, + data: jsonBody, + success: function(response) { + console.log("SUCCESS:", response); + }, + error: function(response, textStatus, errorThrown) { + console.log(errorThrown); + } + }); + } else if (e.target.id === "doneSceneBtn") { + playVideo(); + } else if (e.target.id.includes("merge_scene")) { + let listContainer = document.querySelector(".list-container"); + sceneDetails = JSON.parse(listContainer.dataset.sceneDetails); + + let result = confirm("Do you want to merge current scene with the next?"); + if (result) { + //Logic to delete the item + console.log("inside merge scene"); + let index = e.target.id.indexOf(":") + 1; + let curSceneId = e.target.id.substring(index); + console.log({ curSceneId }); + let curSceneDescElement = document.getElementById("s_d_" + curSceneId); + console.log({ curSceneDescElement }); + + //next Sibling + let nextSibling = document.getElementById(e.target.id).parentElement + .parentElement.nextSibling; + let nextSiblingSceneId = nextSibling.children[0].id; + let indexNextSib = nextSiblingSceneId.indexOf(":") + 1; + nextSiblingSceneId = nextSiblingSceneId.substring(indexNextSib); + let nextSceneData = sceneDetails.filter( + el => el.sc_id === nextSiblingSceneId + )[0]; + let currSceneData = sceneDetails.filter(el => el.sc_id === curSceneId)[0]; + + if (!currSceneData.deletedScene) { + let deletedSceneArr = []; + deletedSceneArr.push(nextSiblingSceneId); + currSceneData.deletedScene = deletedSceneArr; + } else { + if (nextSceneData.deletedScene) { + currSceneData.deletedScene.push(...nextSceneData.deletedScene); + } + + currSceneData.deletedScene.push(nextSiblingSceneId); + } + + console.log({ nextSceneData, currSceneData }); + + let newStartTime = currSceneData.start_time; + let newEndTime = nextSceneData.finish_time; + currSceneData.finish_time = newEndTime; + + document.getElementById( + `time:${curSceneId}` + ).innerHTML = `${millisToMinutesAndSeconds( + newStartTime + )} to ${millisToMinutesAndSeconds(newEndTime)} minutes`; + // console.log({allSceneArr}); + console.log({ nextSiblingSceneId }); + let nextSceneDescElement = document.getElementById( + "s_d_" + nextSiblingSceneId + ); + console.log({ nextSceneDescElement }); + + curSceneDescElement.value += "\n" + nextSceneDescElement.value; + // document.getElementById(curSceneDescElement+'.'+'txt-scene-description').autogrow(); + nextSibling.remove(); + //delete next scene + let jsonBody = { + sceneId: nextSiblingSceneId + }; + + console.log(e.target.id); + console.log({ "Inside merge:": sceneDetails }); + listContainer.dataset.sceneDetails = JSON.stringify(sceneDetails); + updateAll(); + fetchVideoInfo(youtubeId); + // let currentTime = player.getCurrentTime(); + // player.seekTo(currentTime); + } + } +}); + +function millisToMinutesAndSeconds(secs) { + var minutes = Math.floor(secs / 60); + var seconds = ((secs % 60) / 1).toFixed(1); + return minutes + ":" + seconds; +} + +function scrollToPosition(position) { + position--; + $("#ul-scene-list").animate( + { + scrollTop: + $("#ul-scene-list>.newscene-list:eq(" + position + ")").position().top - + $("#ul-scene-list>.newscene-list:eq(0)").position().top + }, + "slow" + ); + let newSceneList = document.querySelectorAll(".newscene-list"); + + newSceneList.forEach(sceneItem => { + sceneItem.firstChild.classList.remove("highlight"); + }); + + newSceneList[position].firstChild.classList.add("highlight"); + + // if(position > 0) { + // newSceneList[position-1].firstChild.classList.remove('highlight'); + // } + console.log({ position }); + console.log({ previouslyHighlightedContainerId }); + previouslyHighlightedContainerId = + "#ul-scene-list>.newscene-list:eq(" + position + ")" + " .scene-container"; +} + +function onYouTubeIframeAPIReady() { + player = new YT.Player("player", { + height: "270", + width: "480", + videoId: youtubeId, + playerVars: { + autoplay: 1, + enablejsapi: 1, + cc_load_policy: 1, + controls: 1, + fs: 0, + iv_load_policy: 3, + modestbranding: 1, + rel: 0, + showinfo: 0, + wmode: "opaque" + }, + events: { + onReady: onPlayerReady, + onStateChange: onPlayerStateChange + } + }); +} + +// function msleep(n){ +// Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, n); +// // Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, n); +// } + +async function onPlayerReady(event) { + pauseVideo(); + player.setVolume(75); + $("#total_time").text(formatTime(player.getDuration())); + // timeupdater = setInterval(updateTime, 100); + // while((!desc_flag && !audio_flag)) + // await new Promise(r => setTimeout(r, 100)); + // playVideo(); + // await new Promise(r => setTimeout(r, 500)); + updateTime(); + document.getElementById("player").tabIndex = -1; + // console.log("player state"); +} + +async function updateTime() { + while (true) { + if (desc_flag) { + var oldTime = videotime; + if (player && player.getCurrentTime) { + videotime = player.getCurrentTime(); + // console.log("No state change"); + } + + if (prevSceneNum >= 0 && videotime - oldTime > 0.3) { + console.log("Forward"); + STATE = 0; + await updateProgress(videotime); // S1 Different func + } else if (oldTime - videotime > 0.3) { + console.log("Rewind"); + STATE = 0; + await updateProgress(videotime); // S1 Different func + } else if (YT_FLAG) YT_FLAG = false; + else if (videotime !== oldTime) { + console.log( + "Playing: oldtime: " + + oldTime + + " videotime: " + + videotime + + " State time: " + + STATE_TIME + ); // S2 Different func + if (STATE == 0) { + // currAudio.play(); + await onProgress(videotime); + } else if (STATE == 1) { + if (YT_FLAG_EXT) { + YT_FLAG_EXT = false; + continue; + } + + pauseVideo(); + YT.currentTime = STATE_TIME; + YT_FLAG = true; + console.log("Extended fault"); + if (currAudio && currAudio !== null) currAudio.pause(); + // currAudio.play(); + console.log("Current Audio is paused: 0" + STATE_TIME); + STATE = 2; + } else if (STATE == 2) { + pauseVideo(); + YT.currentTime = STATE_TIME; + YT_FLAG = true; + videotime = player.getCurrentTime(); + if (currAudio && currAudio !== null) currAudio.play(); + STATE = 1; + } + } else { + // console.log("Paused"); // modify currAudio.onended func + + if ( + STATE == 0 && + currAudio && + currAudio !== null && + isPlaying(currAudio) + ) + currAudio.pause(); + } + $("#elapsed_time").text(formatTime(player.getCurrentTime())); + console.log("Time event"); + // msleep(100); + } + await new Promise(r => setTimeout(r, 100)); + } +} + +// when the time changes, this will be called. + +async function updateProgress(currentTime) { + currentTime = currentTime.toFixed(3); + let currScene = getSceneFromTime(currentTime, allSceneArr); + // let sceneId = currScene.sc_id; + // console.log("The scene id : " + sceneId); + let sceneNum = currScene.sc_num; + // let sceneStartTime = currScene.start_time; + // let sceneEndTime = currScene.finish_time; + // YT_FLAG_Dial = false; + prevSceneNum = sceneNum; + // let currAudTime = currentTime - sceneStartTime; + // if(currAudio !== null && isPlaying(currAudio)) + // {currAudio.pause(); currAudio.currentTime = 0;} + // currAudio = audioArr[sceneNum]; + + // if(currAudio.duration > currAudTime){ + // currAudio.currentTime = currAudTime; + + // if(player.getPlayerState() == YT.PlayerState.PAUSED) + // currAudio.pause(); + // else + // currAudio.play(); // May have to check YT_STATE + // currAudio.onended = function() { + // console.log("This curr Audio ended"); + // if(player.getPlayerState() == YT.PlayerState.PAUSED) { + // STATE = 0; + // playVideo(); + // } + // currAudio = null; + // } + // } + // else + // currAudio = null; + if (currAudio !== null && isPlaying(currAudio)) currAudio.pause(); + currAudio = null; + player.setVolume(75); +} + +async function onProgress(currentTime) { + // debugger; + currentTime = currentTime.toFixed(3); + let currScene = getSceneFromTime(currentTime); + let sceneId = currScene.sc_id; + console.log(allSceneArr); + let sceneNum = currScene.sc_num; + let sceneEndTime = currScene.finish_time; + // if(sceneNum == allSceneArr[allSceneArr.length - 1].sc_num && (sceneEndTime - 1.5 < player.getCurrentTime())) + // sceneNum += 1; + + if (prevSceneNum == -1 && first_extend_flag) { + first_extend_flag = false; + } else if (prevSceneNum != -1 && sceneNum != prevSceneNum) { + if (scene_audio_extend_set.has(prevSceneNum)) { + STATE = 1; + pauseVideo(); + let totalAudios = sentenceAudioMap[prevSceneNum]; + let flag = true; + let i = 0; + YT_FLAG = true; + if (flag && currAudio == null) { + currAudio = sentenceAudioMap[prevSceneNum][0]; + console.log( + "Will play now!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + ); + player.setVolume(15); + currAudio.play(); + currAudio.onended = function() { + console.log("player state is: " + player.getPlayerState()); + if (player.getPlayerState() == YT.PlayerState.PAUSED) { + console.log("I was called"); + STATE = 0; + STATE_TIME = -1; + currAudio = null; + player.setVolume(75); + playVideo(); + } + }; + // while(i setTimeout(r, 100)); + + // } + + prevSceneNum = sceneNum; + + // YT_FLAG_EXT = true; + STATE = 1; + STATE_CHANGE = 1; + console.log("STATE: EXTENDED"); + + STATE_TIME = player.getCurrentTime(); + // debugger; + // prevSceneNum = sceneNum-1; + } + prevSceneNum = sceneNum; + } + prevSceneNum = sceneNum; + } else if (prevSceneNum == -1) prevSceneNum = sceneNum; + else if (currAudio !== null && !isPlaying(currAudio)) currAudio.play(); + // logic for inline + else { + let res = isInlineAudio(currentTime); + let index = res[1]; + if (res[0] && currAudio == null && index >= 0) { + currAudio = allSentenceInlineArr[index]; + let diff = res[2]; + currAudio.currentTime = diff; + player.setVolume(15); + currAudio.play(); + currAudio.onended = function() { + currAudio = null; + console.log("Test1: ", index); + player.setVolume(75); + }; + } + } + // else{ + // // YT_FLAG_Dial = false; + // if(currAudio!==null && !isPlaying(currAudio) && !YT_FLAG_Dial) + // currAudio.play(); + // } +} +/********************************************* */ +// function updateTime() { +// var oldTime = videotime; +// if(player && player.getCurrentTime) { +// videotime = player.getCurrentTime(); +// } +// if(videotime !== oldTime) { +// onProgress(videotime); +// } +// $('#elapsed_time').text(formatTime(player.getCurrentTime())); +// } + +// when the time changes, this will be called. +// function onProgress(currentTime) { +// currentTime = currentTime.toFixed(3); +// let currScene = getSceneFromTime(currentTime); +// let sceneId = currScene.sc_id; +// let sceneNum = currScene.sc_num; +// let sceneEndTime = currScene.finish_time; +// if(prevSceneNum != sceneNum) { +// let listContainer = document.querySelector('.list-container'); +// let sceneDetails = JSON.parse(listContainer.dataset.sceneDetails); +// let = sceneIdToListPositionMap = populateSceneIdToListPositionMap(sceneDetails); +// let listPosition = sceneIdToListPositionMap.get(sceneId); +// console.log({sceneIdToListPositionMap, listPosition}); +// scrollToPosition(listPosition); +// } +// prevSceneNum = sceneNum; +// if(timeAtWhichLastPaused > currentTime) { +// timeAtWhichLastPaused = currentTime; +// } +// if(sceneEndTime - currentTime < 0.1 && currentTime - timeAtWhichLastPaused > 0.5 && autoPauseAtEachScene) { +// pauseVideo(); +// timeAtWhichLastPaused = currentTime; +// } +// let sceneElementId = '#sc_cont_' + sceneId; +// // $(previouslyHighlightedContainerId).removeClass('highlight'); +// $(sceneElementId).addClass('highlight'); +// previouslyHighlightedContainerId = sceneElementId; +// } + +// function onProgress(currentTime) { +// let descType = enableAi ? 'ai' : 'non_ai'; +// let randomNum = Math.random(); +// let src = `../audio/${youtubeId}/${descType}/${userId}/1.mp3` + '?random='+randomNum; +// console.log('src: '+src); +// currentTime = currentTime.toFixed(3); +// let currScene = getSceneFromTime(currentTime); +// let sceneId = currScene.sc_id; +// let sceneNum = currScene.sc_num -1; +// let sceneEndTime = currScene.finish_time; +// if(prevSceneNum != sceneNum) { +// if((currAudio !== null) && (sceneNum !==0) && (isPlaying(currAudio))) { +// pauseVideo(); +// prevSceneNum = sceneNum-1; +// } else { +// let listPosition = sceneIdToListPositionMap.get(sceneId); +// scrollToPosition(listPosition); + +// currAudio = audioArr[sceneNum]; +// currAudio.play(); +// currAudio.onended = function() { +// console.log('player state is: ' + player.getPlayerState()); +// if(player.getPlayerState() == YT.PlayerState.PAUSED) { +// console.log('I was called'); +// playVideo(); +// } else { +// prevSceneNum = sceneNum; +// } +// }; +// prevSceneNum = sceneNum; +// } +// } +// //prevSceneNum = sceneNum; +// if(timeAtWhichLastPaused > currentTime) { +// timeAtWhichLastPaused = currentTime; +// } +// if((currAudio !== null) && (sceneNum !==0) && (isPlaying(currAudio)) && sceneEndTime - currentTime < 0.1 && currentTime - timeAtWhichLastPaused > 0.5 && autoPauseAtEachScene) { +// pauseVideo(); +// timeAtWhichLastPaused = currentTime; +// //if(sceneToBlockMap.has(sceneId)) {//Check if it's the last scene of the current block +// // let blockId = sceneToBlockMap.get(sceneId); +// // let blockElementId = '#bl_cont_' + blockId; +// } +// let sceneElementId = '#sc_cont_' + sceneId; +// $(previouslyHighlightedContainerId).removeClass('highlight'); +// $(sceneElementId).addClass('highlight'); +// previouslyHighlightedContainerId = sceneElementId; +// } + +function onPlayerStateChange(event) { + let playerState = event.data; + switch (playerState) { + case YT.PlayerState.PLAYING: + console.log("i was called: playing state"); + $("#btn-play-pause").html( + '' + ); + break; + case YT.PlayerState.PAUSED: + $("#btn-play-pause").html( + '' + ); + timeAtWhichLastPaused = player.getCurrentTime(); + console.log(timeAtWhichLastPaused); + console.log("i was called: paused state"); + break; + case YT.PlayerState.UNSTARTED: + case YT.PlayerState.ENDED: + if (scene_audio_extend_set.has(sceneNum)) { + // STATE = 1; + // pauseVideo(); + // let totalAudios = sentenceAudioMap[prevSceneNum]; + let flag = true; + // let i=0; + // YT_FLAG = true; + if (flag && currAudio == null) { + currAudio = sentenceAudioMap[sceneNum][0]; + console.log( + "Will play now!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + ); + player.setVolume(15); + currAudio.play(); + currAudio.onended = function() { + console.log("player state is: " + player.getPlayerState()); + // if(player.getPlayerState() == YT.PlayerState.PAUSED) { + // console.log('I was called'); + // // STATE = 0; + // // STATE_TIME = -1; + // currAudio = null; + // player.setVolume(75); + // // playVideo(); + // } + }; + } + } + case YT.PlayerState.CUED: + $("#btn-play-pause").html( + '' + ); + timeAtWhichLastPaused = player.getCurrentTime(); + console.log("i was called: cued state"); + break; + } +} + +function playVideo() { + player.playVideo(); +} + +function pauseVideo() { + console.log("This is the file"); + player.pauseVideo(); +} + +function stopVideo() { + player.stopVideo(); +} + +/* +created by Vaishali +iterating over all QnAcontainers, saving entire info for the ques and answers that already exists +in the DB, sending new qnAs +*/ +const getNewQnAData = () => { + const qnaData = {}; + var allQAContainer = $('div[id^="qna_container:"]'); + + for (let i = 0; i < allQAContainer.length; i++) { + const qaList = allQAContainer[i].children; + for (let childIndex = 0; childIndex < qaList.length; childIndex++) { + console.log("id of parent", allQAContainer[i].id.substring(14)); + let sceneId = allQAContainer[i].id.substring(14); + const currentElement = qaList[childIndex]; + var question = currentElement.children[0].children[1].value; + var answer = currentElement.children[0].children[3].value; + console.log({ question, answer }); + + qnaData[sceneId] = qnaData[sceneId] || []; + if (currentElement.dataset.jsonData) { + const tempData = JSON.parse(currentElement.dataset.jsonData); //saving coming json data for the exisiting qnA + qnaData[sceneId].push({ + question, + answer, + answerId: tempData.answerId, + questionId: tempData.questionId + }); + } else { + qnaData[sceneId].push({ videoId: youtubeId, question, answer }); //pushing only questio and answer value to the exisiting qnA. + } + } + } + return qnaData; +}; + +async function updateAll(shoudShowAlert) { + let postJson = fetchJsonBodyForAllTextFields(userId); + const qnaData = getNewQnAData(); + postJson.qnaData = qnaData; + postJson.videoId = youtubeId; + console.log({ HERE: postJson }); + try { + await $.post("/saveAiDescription", postJson, function(response) { + if (shoudShowAlert) { + if (response.success) { + alert( + "Submitted Successfully\nYou will be redirected to the Video List Page" + ); + window.history.back(); + } else { + console.log(response); + alert("Update Status: " + response.success); + } + } + }); + } catch (e) { + console.log(e); + response.send(e); + } +} + +//added start time, end time and has Ai for saving in descrption table +function fetchJsonBodyForAllTextFields(userId) { + let listContainer = document.querySelector(".list-container"); + sceneDetails = JSON.parse(listContainer.dataset.sceneDetails); + + let parentJsonBody = {}; + let sceneArrJson = []; + + console.log({ "INSIDE fetchJsonBodyForAllTextFields ": sceneDetails }); + let currScene; + for (let i = 0; i < sceneDetails.length; i++) { + currScene = sceneDetails[i]; + let currSceneId = currScene.sc_id; + let currSceneDescTextId = "#s_d_" + currSceneId; + let currSceneOcrTextId = "#s_o_" + currSceneId; + let sceneJsonObj = {}; + sceneJsonObj.scene_id = currSceneId; + let csTestbox = $(currSceneDescTextId); + if (csTestbox && csTestbox.val()) { + sceneJsonObj.modified_description = csTestbox.val().trim(); + sceneJsonObj.modified_ocr = $(currSceneOcrTextId) + .val() + .trim(); + sceneJsonObj.start_time = currScene.start_time; + sceneJsonObj.end_time = currScene.finish_time; + sceneJsonObj.has_ai = enableAi; + sceneJsonObj.deletedScene = currScene.deletedScene; + sceneArrJson.push(sceneJsonObj); + } + } + + parentJsonBody.user_id = userId; + parentJsonBody.scene_arr = sceneArrJson; + parentJsonBody.time_taken = timeElapsedOnPage; + + console.log({ parentJsonBody }); + return parentJsonBody; +} + +async function updateMainTables() { + let postJson = fetchJsonBodyForAllTextFields2(allSceneArr, allBlockArr); + + console.log({ postJson }); + await $.post("/updateBlockAndScene", postJson, function(response) { + if (response.success) { + window.alert("Updated Successfully"); + } else { + window.alert("Update Failed"); + } + }); +} + +function fetchJsonBodyForAllTextFields2(allSceneArr, allBlockArr, userId) { + let parentJsonBody = {}; + let sceneArrJson = []; + let blockArrJson = []; + for (let i = 0; i < allSceneArr.length; i++) { + let currScene = allSceneArr[i]; + let currSceneId = currScene.sc_id; + let currSceneDescTextId = "#s_d_" + currSceneId; + let currSceneOcrTextId = "#s_o_" + currSceneId; + let sceneJsonObj = {}; + sceneJsonObj.scene_id = currSceneId; + sceneJsonObj.original_description = $(currSceneDescTextId) + .val() + .trim(); + sceneJsonObj.original_ocr = $(currSceneOcrTextId) + .val() + .trim(); + sceneArrJson.push(sceneJsonObj); + } + for (let i = 0; i < allBlockArr.length; i++) { + let currBlock = allBlockArr[i]; + let currBlockId = currBlock.block_id; + let currBlockDescTextId = "#b_d_" + currBlockId; + let blockJsonObj = {}; + blockJsonObj.block_id = currBlockId; + blockJsonObj.original_description = $(currBlockDescTextId) + .val() + .trim(); + blockArrJson.push(blockJsonObj); + } + parentJsonBody.scene_arr = sceneArrJson; + parentJsonBody.block_arr = blockArrJson; + return parentJsonBody; +} + +function populateSceneIdToListPositionMap(sceneDetailsArr) { + let listPositionIndex = 0; + let sceneListPositionMap = new Map(); + + for (let i = 0; i < sceneDetailsArr.length; i++) { + let currSceneId = sceneDetailsArr[i].sc_id; + sceneListPositionMap.set(currSceneId, ++listPositionIndex); //sceneToBlockMap + } + return sceneListPositionMap; +} + +function populateSceneIdToListPosition(blocksJsonArr) { + let listPositionIndex = 0; + let sceneListPositionMap = new Map(); + for (let i = 0; i < blocksJsonArr.length; i++) { + listPositionIndex++; + let currBlock = blocksJsonArr[i]; + let currBlockId = currBlock.block_id; + let childSceneArr = currBlock.child_scenes; + for (let j = 0; j < childSceneArr.length; j++) { + listPositionIndex++; + let currSceneId = childSceneArr[j].sc_id; + sceneListPositionMap.set(currSceneId, listPositionIndex); //sceneToBlockMap + if (j == childSceneArr.length - 1) { + sceneToBlockMap.set(currSceneId, currBlockId); + } + } + listPositionIndex++; + } + desc_flag = true; + return sceneListPositionMap; +} + +function getSceneFromTime(currentTime) { + let listContainer = document.querySelector(".list-container"); + console.log("This: " + listContainer.dataset); + sceneDetails = JSON.parse(listContainer.dataset.sceneDetails); + + let startTimesArr = []; + for (let i = 0; i < sceneDetails.length; i++) { + let startTime = parseFloat(sceneDetails[i].start_time); + startTimesArr.push(startTime); + } + let sceneIndex = getIndexOfInputArrForTargetVal(currentTime, startTimesArr); + return sceneDetails[sceneIndex]; +} + +function getSceneFromTime2(currentTime, sceneArr) { + let startTimesArr = []; + for (let i = 0; i < sceneArr.length; i++) { + let startTime = parseFloat(sceneArr[i].start_time); + startTimesArr.push(startTime); + } + let sceneIndex = getIndexOfInputArrForTargetVal(currentTime, startTimesArr); + return sceneArr[sceneIndex]; +} +// Convert to Binary search +function getDialogFromTime(currentTime) { + let index = allDialogArraySTartTimes.length - 1; + for (let i = 0; i < allDialogArraySTartTimes.length; i++) { + if (Number(currentTime) < allDialogArraySTartTimes[i]) { + index = i - 1; + console.log("************************************** " + i); + break; + } + } + // console.log("Dialog times: i: "+i+" currentTime: "+currentTime); + console.log( + "cTime: " + + typeof currentTime + + " typeof Dialog: " + + typeof allDialogArraySTartTimes[4] + + " i: " + + index + ); + if (Number(currentTime) < allDialogArrayEndTimes[index]) { + console.log("##########################################################"); + return true; + } else return false; +} + +//Returns index of array item whose value is equal-to or lesser-than(also nearest-in-value) the target (binary search algorithm) +function getIndexOfInputArrForTargetVal(target, inputArr) { + let start = 0; + let end = inputArr.length - 1; + let index = -1; + while (start <= end) { + let mid = Math.floor((start + end) / 2); + if (inputArr[mid] == target) { + index = mid; + break; + } else if (inputArr[mid] < target) { + index = mid; + start = mid + 1; + } else { + end = mid - 1; + } + } + return index; +} + +function isInlineAudio(target) { + let inputArr = allSentenceInlineStartTimeArr; + let start = 0; + let end = inputArr.length - 1; + let index = -1; + while (start <= end) { + let mid = Math.floor((start + end) / 2); + if (inputArr[mid] == target) { + index = mid; + break; + } else if (inputArr[mid] < target) { + index = mid; + start = mid + 1; + } else { + end = mid - 1; + } + } + console.log( + "inside inline: ", + target, + " and ", + allSentenceInlineStartTimeArr[index] + allSentenceInlineDurArr[index] + ); + if ( + index < 0 || + target > + allSentenceInlineStartTimeArr[index] + allSentenceInlineDurArr[index] + ) + return [false, index, 0]; + // may return the difference of current time and inline start time + else return [true, index, target - allSentenceInlineStartTimeArr[index]]; +} + +function getUrlVars() { + var vars = {}; + var parts = window.location.href.replace(/[?&]+([^=&]+)=([^&]*)/gi, function( + m, + key, + value + ) { + vars[key] = value; + }); + return vars; +} + +function auto_grow(element) { + element.style.height = "5px"; + element.style.height = element.scrollHeight + "px"; +} + +function resetVideoTo(time) { + let timeInSecs = parseFloat(time); + player.seekTo(timeInSecs); + player.playVideo(); +} + +function formatTime(time) { + var minutes = Math.floor(time / 60); + var seconds = time - minutes * 60; + seconds = seconds < 10 ? "0" + seconds : seconds; + var s = parseFloat(seconds).toFixed(1); + return minutes + ":" + s; +} + +function formatTimeForTimer(time) { + var minutes = Math.floor(time / 60); + var seconds = time - minutes * 60; + seconds = seconds < 10 ? "0" + seconds : seconds; + var s = parseFloat(seconds).toFixed(0); + return minutes + ":" + s; +} + +function onPlayPauseButtonClick() { + let playerState = player.getPlayerState(); + switch (playerState) { + case YT.PlayerState.PLAYING: + pauseVideo(); + break; + case YT.PlayerState.PAUSED: + case YT.PlayerState.UNSTARTED: + case YT.PlayerState.ENDED: + case YT.PlayerState.CUED: + playVideo(); + break; + } +} + +function populateAllSceneArrAndAllBlockArr(blocksJsonArr) { + for (let i = 0; i < blocksJsonArr.length; i++) { + let currBlock = blocksJsonArr[i]; + allBlockArr.push(currBlock); + let childSceneArr = currBlock.child_scenes; + allSceneArr.push.apply(allSceneArr, childSceneArr); + } +} + +function isPlaying(audioElement) { + if (audioElement != null) return !audioElement.paused; +} diff --git a/public/javascripts/onDemandTutorial.js b/public/javascripts/onDemandTutorial.js new file mode 100644 index 0000000..5e0aa55 --- /dev/null +++ b/public/javascripts/onDemandTutorial.js @@ -0,0 +1,599 @@ +let player; +let youtubeId = ""; +let videotime = 0; +let timeupdater = null; +let sceneIdToListPositionMap = new Map(); +let sceneToBlockMap = new Map(); +let userId = "c833874d-e2f1-45b3-afd1-16ac6d7fa0b4"; +let volunteerId; +let prevSceneNum = -1; +let timeAtWhichLastPaused = -1; +let previouslyHighlightedContainerId = "#doesnotexist"; +let enableAi = false; +let autoPauseAtEachScene = true; +let timeElapsedOnPage = 0; +let isTimerPaused = false; +let allSceneArr = []; +let allBlockArr = []; +let audioArr = []; +let startTimesArr = []; +let allDialogArraySTartTimes = []; +let allDialogArrayEndTimes = []; + +let allSentenceStartTimes = []; +let allSentenceEndTimes = []; +let isSceneExtended = []; // Required to check if a scene has to be auto-paused to play the extended audio +let allSentenceAudioType = []; +let allSentenceSceneNum = []; +let scene_audio_extend_set = new Set(); +let sentenceAudioMap = {}; +let allSentenceInlineStartTimeArr = []; +let allSentenceInlineDurArr = []; +let allSentenceInlineArr = []; +let allSentenceExtendedArr = []; +let allSentenceExtendedAudioArr = []; + +let currAudio = null; +let currAudioTime; +let wasCurrAudioPaused = false; +let STATE = 0; +let YT_STATE = 0; +let STATE_TIME = -1; +let STATE_CHANGE = 0; +let YT_FLAG = false; +let YT_FLAG_Dial = false; +let YT_FLAG_EXT = false; +let first_extend_flag = true; +let desc_flag = false; +let audio_flag = false; + +let noOfQuestionsAsked = 0; +let playQuestionAudio = new Audio("../audio/tutorial_question.mp3"); +// let audio_clip = document.getElementById("myAudio"); + +let isTutorial = true; +let tutorialStarted = false; +let currentInstruction = 0; +let playPause = 0; +let captionsRequested = 0; +let questionsAsked = 0; +let isVote = true; +let conditionType = "A"; +var audioVote = document.getElementById("voteAudio"); + +$(document).ready(function() { + $("#btn-video-breakpoint-toggle").prop("checked", true); + volunteerId = getUrlVars()["u"]; + // userId = getUrlVars()["u"]; + enableAi = getUrlVars()["ai"] == "yes" ? true : false; + let href = location.href; + const lastPathSegment = new URL(href).pathname.split("/").pop(); + youtubeId = lastPathSegment; + fetchVideoInfo(youtubeId); + + $("#btn-play-pause").click(function() { + onPlayPauseButtonClick(); + }); + $("#btn-timer").click(function() { + isTimerPaused = !isTimerPaused; + if (isTimerPaused) { + $("#btn-timer").attr("src", "/images/play_white.png"); + } else { + $("#btn-timer").attr("src", "/images/pause_white.png"); + } + }); + setInterval(incrementSeconds, 1000); +}); + +function tutorialProgress() { + if (tutorialSteps[currentInstruction].init) { + tutorialSteps[currentInstruction].init(); + tutorialSteps[currentInstruction].recurringSpeech(); + } + + if (tutorialSteps[currentInstruction].check()) { + currentInstruction += 1; + } +} + +function incrementSeconds() { + if (!isTimerPaused) { + timeElapsedOnPage += 1; + $("#txt-timeElaspsed").html(formatTimeForTimer(timeElapsedOnPage)); + } +} + +function fetchVideoInfo(youtubeId) { + WebSocketTest(); + console.log(userId); + // let targetUrl = '/fetchVideoData' + '?videoid=' + youtubeId; + let targetUrl = `/fetchVideoData?videoid=${youtubeId}&userId=${userId}`; + $.get(targetUrl, function(response) { + let vidJson = response; + let vidTitle = vidJson.title; + $("#txtVidtitle").html("Video Title: " + vidTitle); + desc_flag = true; + }); + + let target_url_sentence = + "/getSentences" + "?videoId=" + youtubeId + "&userId=" + userId; + $.get(target_url_sentence, function(response) { + // KeyFramesArr = response; + // debugger; + console.log("sentence start times" + response[0]); + allSentenceArr = response; + let prevSceneNo = -1; + for (val of response) { + allSentenceStartTimes.push(val["audio_start_time"]); + // allSentenceEndTimes.push(val['audio_end_time']); + allSentenceAudioType.push(val["audio_type"]); + allSentenceSceneNum.push(val["scene_num"]); + // let sceneId = val['scene_id']; + let sceneNum = val["scene_num"]; + if (sceneNum != prevSceneNo) { + startTimesArr.push(val["start_time"]); + prevSceneNo = sceneNum; + } + let randomNum = Math.random(); + // let descType = val['has_ai'] ? 'ai' : 'non_ai'; + let descType = "ai"; + // let sentNumArr = val['sentence_id'].split('_'); + // let sentNum = sentNumArr[sentNumArr.length - 1]; + let audioType = val["audio_type"]; + let path = val["audio_path"]; + // let path = val['audio_path'].split('/'); + // let path_ext = path[path.length - 1]; + // let src = `../audio/${youtubeId}/${descType}/${userId}/${sceneNum}/new/${path_ext}` + '?random='+randomNum; + let src = path + "?random=" + randomNum; + console.log(src); + let audioFile = new Audio(src); + // audioArr.push(audioFile); + + if ( + !scene_audio_extend_set.has(val["scene_num"]) && + val["audio_type"] == "extended" + ) { + scene_audio_extend_set.add(val["scene_num"]); + //create audio object + sentenceAudioMap[val["scene_num"]] = [audioFile]; + allSentenceExtendedArr.push(val["audio_start_time"]); + allSentenceExtendedAudioArr.push(audioFile); + } else if (val["audio_type"] == "extended") { + // create audio object + sentenceAudioMap[val["scene_num"]].push(audioFile); + allSentenceExtendedArr.push(val["audio_start_time"]); + allSentenceExtendedAudioArr.push(audioFile); + } else if (val["audio_type"] == "inline") { + allSentenceInlineStartTimeArr.push(val["audio_start_time"]); + allSentenceInlineDurArr.push(val["audio_length"]); + allSentenceInlineArr.push(audioFile); + } + + // debugger; + } + let j = 0, + num = 0; + let presentScene = allSentenceArr[0]["scene_num"]; + let flag = true; + for (val of allSentenceArr) { + if (val["scene_num"] != presentScene) { + if (flag) isSceneExtended.push(false); + flag = true; + } + + if (flag && val["audio_type"] == "extended") { + isSceneExtended.push(true); + flag = false; + } + } + audio_flag = true; + }); + console.log("Times", allDialogArraySTartTimes); +} + +function millisToMinutesAndSeconds(secs) { + var minutes = Math.floor(secs / 60); + var seconds = ((secs % 60) / 1).toFixed(1); + return minutes + ":" + seconds; +} + +function onYouTubeIframeAPIReady() { + let href = location.href; + const youtubeIframeId = new URL(href).pathname.split("/").pop(); + + player = new YT.Player("player", { + height: "270", + width: "480", + videoId: youtubeIframeId, + playerVars: { + autoplay: 1, + enablejsapi: 1, + cc_load_policy: 1, + controls: 0, + disablekb: 1, + fs: 0, + iv_load_policy: 3, + modestbranding: 1, + rel: 0, + showinfo: 0, + wmode: "opaque" + }, + events: { + onReady: onPlayerReady, + onStateChange: onPlayerStateChange + } + }); +} + +async function onPlayerReady(event) { + pauseVideo(); + player.setVolume(75); + $("#total_time").text(formatTime(player.getDuration())); + updateTime(); + document.getElementById("player").tabIndex = -1; +} + +async function updateTime() { + while (true) { + if (desc_flag) { + var oldTime = videotime; + if (player && player.getCurrentTime) { + videotime = player.getCurrentTime(); + } + // if(Math.floor(videotime/30) <= noOfQuestionsAsked) + // STATE = 0; + + // if(Math.floor(videotime/30) > noOfQuestionsAsked){ + // if(player.getPlayerState() == YT.PlayerState.PLAYING){ + // pauseVideo(); + // await checkQuestion(); + // } + // } + + // else if(prevSceneNum >= 0 && videotime - oldTime > 0.3) { + if (prevSceneNum >= 0 && videotime - oldTime > 0.3) { + console.log("Forward"); + STATE = 0; + await updateProgress(videotime); // S1 Different func + } else if (oldTime - videotime > 0.3) { + console.log("Rewind"); + STATE = 0; + await updateProgress(videotime); // S1 Different func + } else if (YT_FLAG) YT_FLAG = false; + else if (videotime !== oldTime) { + console.log( + "Playing: oldtime: " + + oldTime + + " videotime: " + + videotime + + " State time: " + + STATE_TIME + ); // S2 Different func + if (STATE == 0) { + // currAudio.play(); + await onProgress(videotime); + } else if (STATE == 1) { + if (YT_FLAG_EXT) { + YT_FLAG_EXT = false; + continue; + } + + pauseVideo(); + YT.currentTime = STATE_TIME; + YT_FLAG = true; + console.log("Extended fault"); + if (currAudio && currAudio !== null) currAudio.pause(); + // currAudio.play(); + console.log("Current Audio is paused: 0" + STATE_TIME); + STATE = 2; + } else if (STATE == 2) { + pauseVideo(); + YT.currentTime = STATE_TIME; + YT_FLAG = true; + videotime = player.getCurrentTime(); + if (currAudio && currAudio !== null) currAudio.play(); + STATE = 1; + } + } else { + // console.log("Paused"); // modify currAudio.onended func + + if ( + STATE == 0 && + currAudio && + currAudio !== null && + isPlaying(currAudio) + ) + currAudio.pause(); + } + $("#elapsed_time").text(formatTime(player.getCurrentTime())); + console.log("Time event"); + // msleep(100); + } + await new Promise(r => setTimeout(r, 100)); + } +} + +// when the time changes, this will be called. + +async function updateProgress(currentTime) { + currentTime = currentTime.toFixed(3); + let currScene = getIndexFromExtended(currentTime); + + // let sceneNum = currScene.sc_num; + + prevSceneNum = currScene; + + if (currAudio !== null && isPlaying(currAudio)) currAudio.pause(); + currAudio = null; + player.setVolume(75); +} + +async function onProgress(currentTime) { + // debugger; + currentTime = currentTime.toFixed(3); + // let currScene = getSceneFromTime(currentTime); + let currScene = getIndexFromExtended(currentTime); + // console.log(currScene); + // let sceneNum = currScene.sc_num; + let sceneNum = currScene; + + if (prevSceneNum == -1 && first_extend_flag) { + first_extend_flag = false; + } else if (sceneNum != -1 && sceneNum != prevSceneNum) { + if (true || scene_audio_extend_set.has(prevSceneNum)) { + STATE = 1; + pauseVideo(); + let totalAudios = sentenceAudioMap[prevSceneNum]; + let flag = true; + let i = 0; + YT_FLAG = true; + if (flag && currAudio == null) { + // currAudio = sentenceAudioMap[prevSceneNum][0]; + currAudio = allSentenceExtendedAudioArr[currScene]; + currAudio.currentTime = 0; + player.setVolume(15); + currAudio.play(); + currAudio.onended = function() { + console.log("player state is: " + player.getPlayerState()); + if (player.getPlayerState() == YT.PlayerState.PAUSED) { + STATE = 0; + STATE_TIME = -1; + currAudio = null; + player.setVolume(75); + playVideo(); + } + }; + + prevSceneNum = sceneNum; + + // YT_FLAG_EXT = true; + STATE = 1; + STATE_CHANGE = 1; + console.log("STATE: EXTENDED"); + + STATE_TIME = player.getCurrentTime(); + // debugger; + // prevSceneNum = sceneNum-1; + } + prevSceneNum = sceneNum; + } + prevSceneNum = sceneNum; + } //else if (prevSceneNum == -1) prevSceneNum = sceneNum; + else if (currAudio !== null && !isPlaying(currAudio)) currAudio.play(); + // logic for inline + else { + let res = isInlineAudio(currentTime); + let index = res[1]; + if (res[0] && currAudio == null && index >= 0) { + currAudio = allSentenceInlineArr[index]; + let diff = res[2]; + currAudio.currentTime = diff; + player.setVolume(15); + currAudio.play(); + currAudio.onended = function() { + currAudio = null; + console.log("Test1: ", index); + player.setVolume(75); + }; + } + } + // else{ + // // YT_FLAG_Dial = false; + // if(currAudio!==null && !isPlaying(currAudio) && !YT_FLAG_Dial) + // currAudio.play(); + // } +} + +function onPlayerStateChange(event) { + let playerState = event.data; + switch (playerState) { + case YT.PlayerState.PLAYING: + audio_clip.pause(); + isChatbotInProcess = false; + isQuestionInProcess = false; + console.log("i was called: playing state"); + $("#btn-play-pause").html( + '' + ); + break; + case YT.PlayerState.PAUSED: + $("#btn-play-pause").html( + '' + ); + timeAtWhichLastPaused = player.getCurrentTime(); + console.log(timeAtWhichLastPaused); + console.log("i was called: paused state"); + break; + case YT.PlayerState.UNSTARTED: + case YT.PlayerState.ENDED: + if (scene_audio_extend_set.has(sceneNum)) { + // STATE = 1; + // pauseVideo(); + // let totalAudios = sentenceAudioMap[prevSceneNum]; + let flag = true; + // let i=0; + // YT_FLAG = true; + if (flag && currAudio == null) { + currAudio = sentenceAudioMap[sceneNum][0]; + console.log( + "Will play now!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + ); + player.setVolume(15); + currAudio.play(); + currAudio.onended = function() { + console.log("player state is: " + player.getPlayerState()); + }; + } + } + case YT.PlayerState.CUED: + $("#btn-play-pause").html( + '' + ); + timeAtWhichLastPaused = player.getCurrentTime(); + console.log("i was called: cued state"); + break; + } +} + +function playVideo() { + // if (tutorialStarted) { + //speak("Video Play"); + player.playVideo(); + // } +} + +function pauseVideo() { + // if (tutorialStarted) { + //speak("Video Play"); + // playPause += 1; + // console.log("This is the file"); + player.pauseVideo(); + // } +} + +function stopVideo() { + player.stopVideo(); +} + +function getSceneFromTime(currentTime) { + let listContainer = document.querySelector(".list-container"); + console.log("This: " + listContainer.dataset); + sceneDetails = JSON.parse(listContainer.dataset.sceneDetails); + + // let startTimesArr = []; + // for(let i=0; i< sceneDetails.length; i++) { + // let startTime = parseFloat(sceneDetails[i].start_time); + // startTimesArr.push(startTime); + // } + let sceneIndex = getIndexOfInputArrForTargetVal(currentTime, startTimesArr); + return sceneDetails[sceneIndex]; +} + +function getIndexFromExtended(currentTime) { + let index = getIndexOfInputArrForTargetVal( + currentTime, + allSentenceExtendedArr + ); + console.log(index); + return index; +} + +//Returns index of array item whose value is equal-to or lesser-than(also nearest-in-value) the target (binary search algorithm) +function getIndexOfInputArrForTargetVal(target, inputArr) { + let start = 0; + let end = inputArr.length - 1; + let index = -1; + while (start <= end) { + let mid = Math.floor((start + end) / 2); + if (inputArr[mid] == target) { + index = mid; + break; + } else if (inputArr[mid] < target) { + index = mid; + start = mid + 1; + } else { + end = mid - 1; + } + } + return index; +} + +function isInlineAudio(target) { + let inputArr = allSentenceInlineStartTimeArr; + let start = 0; + let end = inputArr.length - 1; + let index = -1; + while (start <= end) { + let mid = Math.floor((start + end) / 2); + if (inputArr[mid] == target) { + index = mid; + break; + } else if (inputArr[mid] < target) { + index = mid; + start = mid + 1; + } else { + end = mid - 1; + } + } + console.log( + "inside inline: ", + target, + " and ", + allSentenceInlineStartTimeArr[index] + allSentenceInlineDurArr[index] + ); + if ( + index < 0 || + target > + allSentenceInlineStartTimeArr[index] + allSentenceInlineDurArr[index] + ) + return [false, index, 0]; + // may return the difference of current time and inline start time + else return [true, index, target - allSentenceInlineStartTimeArr[index]]; +} + +function getUrlVars() { + var vars = {}; + var parts = window.location.href.replace(/[?&]+([^=&]+)=([^&]*)/gi, function( + m, + key, + value + ) { + vars[key] = value; + }); + return vars; +} + +function formatTime(time) { + var minutes = Math.floor(time / 60); + var seconds = time - minutes * 60; + seconds = seconds < 10 ? "0" + seconds : seconds; + var s = parseFloat(seconds).toFixed(1); + return minutes + ":" + s; +} + +function formatTimeForTimer(time) { + var minutes = Math.floor(time / 60); + var seconds = time - minutes * 60; + seconds = seconds < 10 ? "0" + seconds : seconds; + var s = parseFloat(seconds).toFixed(0); + return minutes + ":" + s; +} + +function onPlayPauseButtonClick() { + let playerState = player.getPlayerState(); + switch (playerState) { + case YT.PlayerState.PLAYING: + pauseVideo(); + break; + case YT.PlayerState.PAUSED: + case YT.PlayerState.UNSTARTED: + case YT.PlayerState.ENDED: + case YT.PlayerState.CUED: + playVideo(); + break; + } +} + +function isPlaying(audioElement) { + if (audioElement != null) return !audioElement.paused; +} diff --git a/public/javascripts/onDemandUserstudy.js b/public/javascripts/onDemandUserstudy.js new file mode 100644 index 0000000..0e3c853 --- /dev/null +++ b/public/javascripts/onDemandUserstudy.js @@ -0,0 +1,647 @@ +let player; +let youtubeId = ""; +let videotime = 0; +let timeupdater = null; +let sceneIdToListPositionMap = new Map(); +let sceneToBlockMap = new Map(); +// let userId = "d8fcfd52-b0b3-4969-bf03-1dc54b681b5d"; +let userId = "f03cb948-474b-4084-9192-650ba62d396b"; +let volunteerId; +let prevSceneNum = -1; +let lastSceneNum = -1; +let timeAtWhichLastPaused = -1; +let previouslyHighlightedContainerId = "#doesnotexist"; +let enableAi = false; +let baselineDesc = "and Baseline Descriptions"; +let autoPauseAtEachScene = true; +let timeElapsedOnPage = 0; +let isTimerPaused = false; +let allSceneArr = []; +let allBlockArr = []; +let audioArr = []; +let startTimesArr = []; +let allDialogArraySTartTimes = []; +let allDialogArrayEndTimes = []; + +let allSentenceStartTimes = []; +let allSentenceEndTimes = []; +let isSceneExtended = []; // Required to check if a scene has to be auto-paused to play the extended audio +let allSentenceAudioType = []; +let allSentenceSceneNum = []; +let scene_audio_extend_set = new Set(); +let sentenceAudioMap = {}; +let allSentenceInlineStartTimeArr = []; +let allSentenceInlineDurArr = []; +let allSentenceInlineArr = []; +let allSentenceExtendedArr = []; +let allSentenceExtendedAudioArr = []; + +let currAudio = null; +let currAudioTime; +let wasCurrAudioPaused = false; +let STATE = 0; +let YT_STATE = 0; +let STATE_TIME = -1; +let STATE_CHANGE = 0; +let YT_FLAG = false; +let YT_FLAG_Dial = false; +let YT_FLAG_EXT = false; +let first_extend_flag = true; +let desc_flag = false; +let audio_flag = false; +let noOfQuestionsAsked = 0; +let isTutorial = false; +let isVote = true; +let conditionType = ""; +var audioVote = document.getElementById("voteAudio"); + +$(document).ready(function() { + $("#btn-video-breakpoint-toggle").prop("checked", true); + volunteerId = getUrlVars()["u"]; + // userId = getUrlVars()["u"]; + describerId = getUrlVars()["did"]; + userId = + describerId == "1" + ? "f03cb948-474b-4084-9192-650ba62d396b" + : "c833874d-e2f1-45b3-afd1-16ac6d7fa0b4"; + enableAi = getUrlVars()["ai"] == "yes" ? true : false; + if (!enableAi) { + userId = ""; + baselineDesc = "Descriptions"; + conditionType = "C"; + document.getElementById("sidebar-text").innerHTML = chatBotText; + } else { + conditionType = describerId == "1" ? "B" : "A"; + document.getElementById("sidebar-text").innerHTML = baseLineChatBotText; + } + + document.getElementById("page-title").innerHTML = "InfoBot " + baselineDesc; + + let href = location.href; + const lastPathSegment = new URL(href).pathname.split("/").pop(); + youtubeId = lastPathSegment; + fetchVideoInfo(youtubeId); + + $("#btn-play-pause").click(function() { + onPlayPauseButtonClick(); + }); + $("#btn-timer").click(function() { + isTimerPaused = !isTimerPaused; + if (isTimerPaused) { + $("#btn-timer").attr("src", "/images/play_white.png"); + } else { + $("#btn-timer").attr("src", "/images/pause_white.png"); + } + }); + setInterval(incrementSeconds, 1000); +}); + +function incrementSeconds() { + if (!isTimerPaused) { + timeElapsedOnPage += 1; + $("#txt-timeElaspsed").html(formatTimeForTimer(timeElapsedOnPage)); + } +} + +function fetchVideoInfo(youtubeId) { + WebSocketTest(); + console.log(userId); + // let targetUrl = '/fetchVideoData' + '?videoid=' + youtubeId; + let targetUrl = `/fetchVideoData?videoid=${youtubeId}&userId=${userId}`; + $.get(targetUrl, function(response) { + let vidJson = response; + let vidTitle = vidJson.title; + $("#txtVidtitle").html(vidTitle); + desc_flag = true; + }); + + let target_url_sentence = + "/getSentences" + "?videoId=" + youtubeId + "&userId=" + userId; + $.get(target_url_sentence, function(response) { + // KeyFramesArr = response; + // debugger; + console.log("sentence start times" + response[0]); + allSentenceArr = response; + let prevSceneNo = -1; + for (val of response) { + allSentenceStartTimes.push(val["audio_start_time"]); + // allSentenceEndTimes.push(val['audio_end_time']); + allSentenceAudioType.push(val["audio_type"]); + allSentenceSceneNum.push(val["scene_num"]); + // let sceneId = val['scene_id']; + let sceneNum = val["scene_num"]; + if (sceneNum != prevSceneNo) { + startTimesArr.push(val["start_time"]); + prevSceneNo = sceneNum; + } + let randomNum = Math.random(); + // let descType = val['has_ai'] ? 'ai' : 'non_ai'; + let descType = "ai"; + // let sentNumArr = val['sentence_id'].split('_'); + // let sentNum = sentNumArr[sentNumArr.length - 1]; + let audioType = val["audio_type"]; + let path = val["audio_path"]; + // let path = val['audio_path'].split('/'); + // let path_ext = path[path.length - 1]; + // let src = `../audio/${youtubeId}/${descType}/${userId}/${sceneNum}/new/${path_ext}` + '?random='+randomNum; + let src = path + "?random=" + randomNum; + console.log(src); + let audioFile = new Audio(src); + // audioArr.push(audioFile); + + if ( + !scene_audio_extend_set.has(val["scene_num"]) && + val["audio_type"] == "extended" + ) { + scene_audio_extend_set.add(val["scene_num"]); + //create audio object + sentenceAudioMap[val["scene_num"]] = [audioFile]; + allSentenceExtendedArr.push(val["audio_start_time"]); + allSentenceExtendedAudioArr.push(audioFile); + } else if (val["audio_type"] == "extended") { + // create audio object + sentenceAudioMap[val["scene_num"]].push(audioFile); + allSentenceExtendedArr.push(val["audio_start_time"]); + allSentenceExtendedAudioArr.push(audioFile); + } else if (val["audio_type"] == "inline") { + allSentenceInlineStartTimeArr.push(val["audio_start_time"]); + allSentenceInlineDurArr.push(val["audio_length"]); + allSentenceInlineArr.push(audioFile); + } + + // debugger; + } + let j = 0, + num = 0; + let presentScene = allSentenceArr[0]["scene_num"]; + let flag = true; + for (val of allSentenceArr) { + if (val["scene_num"] != presentScene) { + if (flag) isSceneExtended.push(false); + flag = true; + } + + if (flag && val["audio_type"] == "extended") { + isSceneExtended.push(true); + flag = false; + } + } + audio_flag = true; + }); + console.log("Times", allDialogArraySTartTimes); +} + +function millisToMinutesAndSeconds(secs) { + var minutes = Math.floor(secs / 60); + var seconds = ((secs % 60) / 1).toFixed(1); + return minutes + ":" + seconds; +} + +function onYouTubeIframeAPIReady() { + let href = location.href; + const youtubeIframeId = new URL(href).pathname.split("/").pop(); + + player = new YT.Player("player", { + height: "270", + width: "480", + videoId: youtubeIframeId, + playerVars: { + autoplay: 1, + enablejsapi: 1, + cc_load_policy: 1, + controls: 0, + disablekb: 1, + fs: 0, + iv_load_policy: 3, + rel: 0, + showinfo: 0, + wmode: "opaque" + }, + events: { + onReady: onPlayerReady, + onStateChange: onPlayerStateChange + } + }); +} + +async function onPlayerReady(event) { + pauseVideo(); + player.setVolume(75); + $("#total_time").text(formatTime(player.getDuration())); + updateTime(); + document.getElementById("player").tabIndex = -1; +} + +async function updateTime() { + while (true) { + if (desc_flag) { + var oldTime = videotime; + if (player && player.getCurrentTime) { + videotime = player.getCurrentTime(); + // console.log("No state change"); + } + // if(Math.floor(videotime/30) <= noOfQuestionsAsked) + // STATE = 0; + + // if(Math.floor(videotime/30) > noOfQuestionsAsked){ + // if(player.getPlayerState() == YT.PlayerState.PLAYING){ + // pauseVideo(); + // await checkQuestion(); + // } + // } + + // else if(prevSceneNum >= 0 && videotime - oldTime > 0.3) { + if (prevSceneNum >= 0 && videotime - oldTime > 0.3) { + console.log("Forward"); + STATE = 0; + await updateProgress(videotime); // S1 Different func + } else if (oldTime - videotime > 0.3) { + console.log("Rewind"); + STATE = 0; + await updateProgress(videotime); // S1 Different func + } else if (YT_FLAG) YT_FLAG = false; + else if (videotime !== oldTime) { + console.log( + "Playing: oldtime: " + + oldTime + + " videotime: " + + videotime + + " State time: " + + STATE_TIME + ); // S2 Different func + if (STATE == 0) { + // currAudio.play(); + await onProgress(videotime); + } else if (STATE == 1) { + if (YT_FLAG_EXT) { + YT_FLAG_EXT = false; + continue; + } + + pauseVideo(); + YT.currentTime = STATE_TIME; + YT_FLAG = true; + console.log("Extended fault"); + if (currAudio && currAudio !== null) currAudio.pause(); + // currAudio.play(); + console.log("Current Audio is paused: 0" + STATE_TIME); + STATE = 2; + } else if (STATE == 2) { + pauseVideo(); + YT.currentTime = STATE_TIME; + YT_FLAG = true; + videotime = player.getCurrentTime(); + if (currAudio && currAudio !== null) currAudio.play(); + STATE = 1; + } + } else { + // console.log("Paused"); // modify currAudio.onended func + + if ( + STATE == 0 && + currAudio && + currAudio !== null && + isPlaying(currAudio) + ) + currAudio.pause(); + } + $("#elapsed_time").text(formatTime(player.getCurrentTime())); + console.log("Time event"); + // msleep(100); + } + await new Promise(r => setTimeout(r, 100)); + } +} + +// when the time changes, this will be called. + +async function updateProgress(currentTime) { + currentTime = currentTime.toFixed(3); + let currScene = getIndexFromExtended(currentTime); + + // let sceneNum = currScene.sc_num; + + prevSceneNum = currScene; + + if (currAudio !== null && isPlaying(currAudio)) currAudio.pause(); + currAudio = null; + player.setVolume(75); +} + +async function onProgress(currentTime) { + // debugger; + currentTime = currentTime.toFixed(3); + // let currScene = getSceneFromTime(currentTime); + let currScene = getIndexFromExtended(currentTime); + // console.log(allSceneArr); + // let sceneNum = currScene.sc_num; + let sceneNum = currScene; + + if (prevSceneNum == -1 && first_extend_flag) { + first_extend_flag = false; + } + + // else if((prevSceneNum != -1 && sceneNum != prevSceneNum)) { + else if (sceneNum != -1 && sceneNum != prevSceneNum) { + if (true || scene_audio_extend_set.has(prevSceneNum)) { + STATE = 1; + pauseVideo(); + let totalAudios = sentenceAudioMap[prevSceneNum]; + let flag = true; + let i = 0; + YT_FLAG = true; + if (flag && currAudio == null) { + // currAudio = sentenceAudioMap[prevSceneNum][0]; + currAudio = allSentenceExtendedAudioArr[currScene]; + console.log( + "Will play now!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + ); + player.setVolume(15); + currAudio.play(); + currAudio.onended = function() { + console.log("player state is: " + player.getPlayerState()); + if (player.getPlayerState() == YT.PlayerState.PAUSED) { + console.log("I was called"); + STATE = 0; + STATE_TIME = -1; + currAudio = null; + player.setVolume(75); + playVideo(); + } + }; + + prevSceneNum = sceneNum; + + // YT_FLAG_EXT = true; + STATE = 1; + STATE_CHANGE = 1; + console.log("STATE: EXTENDED"); + + STATE_TIME = player.getCurrentTime(); + // debugger; + // prevSceneNum = sceneNum-1; + } + prevSceneNum = sceneNum; + } + prevSceneNum = sceneNum; + } + + // else if(prevSceneNum == -1) + // prevSceneNum = sceneNum; + else if (currAudio !== null && !isPlaying(currAudio)) currAudio.play(); + // logic for inline + else { + let res = isInlineAudio(currentTime); + let index = res[1]; + if (res[0] && currAudio == null && index >= 0) { + currAudio = allSentenceInlineArr[index]; + let diff = res[2]; + currAudio.currentTime = diff; + player.setVolume(15); + currAudio.play(); + currAudio.onended = function() { + currAudio = null; + console.log("Test1: ", index); + player.setVolume(75); + }; + } + } + // else{ + // // YT_FLAG_Dial = false; + // if(currAudio!==null && !isPlaying(currAudio) && !YT_FLAG_Dial) + // currAudio.play(); + // } +} + +function onPlayerStateChange(event) { + let playerState = event.data; + switch (playerState) { + case YT.PlayerState.PLAYING: + audio_clip.pause(); + isChatbotInProcess = false; + isQuestionInProcess = false; + console.log("i was called: playing state"); + $("#btn-play-pause").html( + '' + ); + break; + case YT.PlayerState.PAUSED: + $("#btn-play-pause").html( + '' + ); + timeAtWhichLastPaused = player.getCurrentTime(); + console.log(timeAtWhichLastPaused); + console.log("i was called: paused state"); + break; + case YT.PlayerState.UNSTARTED: + case YT.PlayerState.ENDED: + if (scene_audio_extend_set.has(sceneNum)) { + // STATE = 1; + // pauseVideo(); + // let totalAudios = sentenceAudioMap[prevSceneNum]; + let flag = true; + // let i=0; + // YT_FLAG = true; + if (flag && currAudio == null) { + currAudio = sentenceAudioMap[sceneNum][0]; + console.log( + "Will play now!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + ); + player.setVolume(15); + currAudio.play(); + currAudio.onended = function() { + console.log("player state is: " + player.getPlayerState()); + }; + } + } + case YT.PlayerState.CUED: + $("#btn-play-pause").html( + '' + ); + timeAtWhichLastPaused = player.getCurrentTime(); + console.log("i was called: cued state"); + break; + } +} + +function playVideo() { + player.playVideo(); +} + +function pauseVideo() { + console.log("This is the file"); + player.pauseVideo(); +} + +function stopVideo() { + player.stopVideo(); +} + +function getSceneFromTime(currentTime) { + let listContainer = document.querySelector(".list-container"); + console.log("This: " + listContainer.dataset); + sceneDetails = JSON.parse(listContainer.dataset.sceneDetails); + + // let startTimesArr = []; + // for(let i=0; i< sceneDetails.length; i++) { + // let startTime = parseFloat(sceneDetails[i].start_time); + // startTimesArr.push(startTime); + // } + let sceneIndex = getIndexOfInputArrForTargetVal(currentTime, startTimesArr); + return sceneDetails[sceneIndex]; +} + +function getIndexFromExtended(currentTime) { + let index = getIndexOfInputArrForTargetVal( + currentTime, + allSentenceExtendedArr + ); + console.log(index); + return index; +} + +//Returns index of array item whose value is equal-to or lesser-than(also nearest-in-value) the target (binary search algorithm) +function getIndexOfInputArrForTargetVal(target, inputArr) { + let start = 0; + let end = inputArr.length - 1; + let index = -1; + while (start <= end) { + let mid = Math.floor((start + end) / 2); + if (inputArr[mid] == target) { + index = mid; + break; + } else if (inputArr[mid] < target) { + index = mid; + start = mid + 1; + } else { + end = mid - 1; + } + } + return index; +} + +function isInlineAudio(target) { + let inputArr = allSentenceInlineStartTimeArr; + let start = 0; + let end = inputArr.length - 1; + let index = -1; + while (start <= end) { + let mid = Math.floor((start + end) / 2); + if (inputArr[mid] == target) { + index = mid; + break; + } else if (inputArr[mid] < target) { + index = mid; + start = mid + 1; + } else { + end = mid - 1; + } + } + console.log( + "inside inline: ", + target, + " and ", + allSentenceInlineStartTimeArr[index] + allSentenceInlineDurArr[index] + ); + if ( + index < 0 || + target > + allSentenceInlineStartTimeArr[index] + allSentenceInlineDurArr[index] + ) + return [false, index, 0]; + // may return the difference of current time and inline start time + else return [true, index, target - allSentenceInlineStartTimeArr[index]]; +} + +function getUrlVars() { + var vars = {}; + var parts = window.location.href.replace(/[?&]+([^=&]+)=([^&]*)/gi, function( + m, + key, + value + ) { + vars[key] = value; + }); + return vars; +} + +function formatTime(time) { + var minutes = Math.floor(time / 60); + var seconds = time - minutes * 60; + seconds = seconds < 10 ? "0" + seconds : seconds; + var s = parseFloat(seconds).toFixed(1); + return minutes + ":" + s; +} + +function formatTimeForTimer(time) { + var minutes = Math.floor(time / 60); + var seconds = time - minutes * 60; + seconds = seconds < 10 ? "0" + seconds : seconds; + var s = parseFloat(seconds).toFixed(0); + return minutes + ":" + s; +} + +function onPlayPauseButtonClick() { + let playerState = player.getPlayerState(); + switch (playerState) { + case YT.PlayerState.PLAYING: + pauseVideo(); + break; + case YT.PlayerState.PAUSED: + case YT.PlayerState.UNSTARTED: + case YT.PlayerState.ENDED: + case YT.PlayerState.CUED: + playVideo(); + break; + } +} + +function isPlaying(audioElement) { + if (audioElement != null) return !audioElement.paused; +} + +///IGNORE ME I AM JUST TEXT PASSING THROUGH +var baseLineChatBotText = + '
    ' + + " When you watch this video, you will have access to baseline descriptions and the InfoBot. Press the spacebar to start" + + " and stop the video." + + " Baseline descriptions play automatically, provide general information about a scene, and read any onscreen text out" + + " loud. Onscreen text will be read in a female voice." + + "
    " + + " " + + '
    ' + + " On-demand information is accessed using the InfoBot. The InfoBot provides information in two ways: as an image" + + " description or in response to a question." + + "
    " + + '
    ' + + " Press “D” at any time to pause the video and get an image description of the current scene. Click the up or down arrow to give the description a thumbs up (a chime will sound) or thumbs down (a buzzer will sound).. The video will automatically resume playing." + + "
    " + + '
    ' + + " Press “Q” at any time to pause the video and ask questions about the current scene. Press “Q”, then wait for a short beep before asking a question about the current scene. You should receive an answer within 5 seconds. Click the up or down arrow to give the description a thumbs up (a chime will sound) or thumbs down (a buzzer will sound). Ask as many questions as you like by pressing “Q” and asking a question after the beep - Only one question per beep, please! " + + "
    " + + " " + + " " + + "

    " + + " " + + "
    "; + +var chatBotText = + '
    ' + + " When you watch this video, you will only have access to on-demand information." + + "
    " + + " " + + '
    ' + + " On-demand information is accessed using the InfoBot. The InfoBot provides information in two ways: as an image" + + " description or in response to a question." + + "
    " + + '
    ' + + " Press “D” at any time to pause the video and get an image description of the current scene. Click the up or down arrow to give the description a thumbs up (a chime will sound) or thumbs down (a buzzer will sound). The video will automatically resume playing." + + "
    " + + '
    ' + + " Press “Q” at any time to pause the video and ask questions about the current scene. Press “Q”, then wait for a short beep before asking a question about the current scene. You should receive an answer within 5 seconds. Click the up or down arrow to give the description a thumbs up (a chime will sound) or thumbs down (a buzzer will sound). Ask as many questions as you like by pressing “Q” and asking a question after the beep - Only one question per beep, please! " + + "
    " + + " " + + " " + + "

    " + + " " + + "
    "; diff --git a/public/javascripts/playvideodescription.js b/public/javascripts/playvideodescription.js index ae00b51..b238ad3 100644 --- a/public/javascripts/playvideodescription.js +++ b/public/javascripts/playvideodescription.js @@ -9,188 +9,222 @@ let pauseTime = 10000; let currAudioElapsedTime = 0; $(document).ready(function() { - userId = getUrlVars()["u"]; - enableAi = (getUrlVars()["ai"] == 'yes') ? true : false; - let href = location.href; - const lastPathSegment = new URL(href).pathname.split('/').pop(); - youtubeId = lastPathSegment; - fetchDescriptionData(youtubeId, userId, enableAi); - $("#btn-play-pause").click(function() { - //onPlayPauseButtonClick(); - }); + userId = getUrlVars()["u"]; + enableAi = getUrlVars()["ai"] == "yes" ? true : false; + let href = location.href; + const lastPathSegment = new URL(href).pathname.split("/").pop(); + youtubeId = lastPathSegment; + fetchDescriptionData(youtubeId, userId, enableAi); + $("#btn-play-pause").click(function() { + //onPlayPauseButtonClick(); + }); }); function getUrlVars() { - var vars = {}; - var parts = window.location.href.replace(/[?&]+([^=&]+)=([^&]*)/gi, function(m,key,value) { - vars[key] = value; - }); - return vars; + var vars = {}; + var parts = window.location.href.replace(/[?&]+([^=&]+)=([^&]*)/gi, function( + m, + key, + value + ) { + vars[key] = value; + }); + return vars; } function onYouTubeIframeAPIReady() { - player = new YT.Player('player', { - height: '270', - width: '480', - videoId: youtubeId, - playerVars: { - autoplay: 0, - playsinline: 1, - enablejsapi: 1, - cc_load_policy: 1, - controls: 1, - fs: 0, - iv_load_policy: 3, - modestbranding: 1, - rel: 0, - showinfo: 0, - wmode: 'opaque' - }, - events: { - 'onReady': onPlayerReady, - 'onStateChange': onPlayerStateChange - } - }); + player = new YT.Player("player", { + height: "270", + width: "480", + videoId: youtubeId, + playerVars: { + autoplay: 0, + playsinline: 1, + enablejsapi: 1, + cc_load_policy: 1, + controls: 1, + fs: 0, + iv_load_policy: 3, + modestbranding: 1, + rel: 0, + showinfo: 0, + wmode: "opaque" + }, + events: { + onReady: onPlayerReady, + onStateChange: onPlayerStateChange + } + }); } function onPlayerReady(event) { - timeupdater = setInterval(updateTime, 500); + timeupdater = setInterval(updateTime, 500); + document.getElementById("player").tabIndex = -1; } function updateTime() { - var oldTime = videotime; - if(player && player.getCurrentTime) { - videotime = player.getCurrentTime(); - } - if(videotime !== oldTime) { - onProgress(videotime); - } + var oldTime = videotime; + if (player && player.getCurrentTime) { + videotime = player.getCurrentTime(); + } + if (videotime !== oldTime) { + onProgress(videotime); + } + } // when the time changes, this will be called. function onProgress(currentTime) { - if(currentTime > pauseTime) { - player.pauseVideo(); - pauseTime = 10000; - } else { - currentTime = currentTime.toFixed(3); - let currScene = getSceneFromTime(currentTime, startTimesArr); - let sceneNum = currScene.scene_num; - let sceneEndTime = currScene.end_time; - let sceneStartTime = currScene.start_time; - let totalSceneTime = currScene.total_time; - let audioLen = currScene.audio_length; - if(prevSceneNum != sceneNum) { - currAudio = currScene.audio; - if((player.getCurrentTime() - currScene.start_time) < currScene.audio_length) { - currAudio.currentTime = player.getCurrentTime() - currScene.start_time; - currAudio.play(); - if(currScene.total_time < currScene.audio_length) { - pauseTime = currScene.end_time; - currAudio.onended = function() { - if(player.getPlayerState() == YT.PlayerState.PAUSED) { - player.playVideo(); - } - }; - } + if (currentTime > pauseTime) { + player.pauseVideo(); + pauseTime = 10000; + } else { + currentTime = currentTime.toFixed(3); + let currScene = getSceneFromTime(currentTime, startTimesArr); + let sceneNum = currScene.scene_num; + let sceneEndTime = currScene.end_time; + let sceneStartTime = currScene.start_time; + let totalSceneTime = currScene.total_time; + let audioLen = currScene.audio_length; + if (prevSceneNum != sceneNum) { + currAudio = currScene.audio; + if ( + player.getCurrentTime() - currScene.start_time < + currScene.audio_length + ) { + currAudio.currentTime = player.getCurrentTime() - currScene.start_time; + currAudio.play(); + if (currScene.total_time < currScene.audio_length) { + pauseTime = currScene.end_time; + currAudio.onended = function() { + if (player.getPlayerState() == YT.PlayerState.PAUSED) { + player.playVideo(); } + }; } - prevSceneNum = sceneNum; + } } + prevSceneNum = sceneNum; + } } function resumeVideoifInPausedState() { - if (player.getPlayerState() == YT.PlayerState.PAUSED) { - playVideo(); - } + if (player.getPlayerState() == YT.PlayerState.PAUSED) { + playVideo(); + } } function getSceneFromTime(currentTime, startTimesArr) { - let sceneIndex = getIndexOfInputArrForTargetVal(currentTime, startTimesArr); - return sceneMap.get(sceneIndex); + let sceneIndex = getIndexOfInputArrForTargetVal(currentTime, startTimesArr); + return sceneMap.get(sceneIndex); } function fetchDescriptionData(videoId, volunteerId, aiType) { - let targetUrl = `/getDescriptionData?videoId=${videoId}&volunteerId=${volunteerId}&aiType=${aiType}`; - $.get(targetUrl, function (response) { - let descriptionArr = response; - let outputHtml = '' - for(let i=0; i'; - outputHtml = outputHtml + '' + '' + '' + '' + '' + ''; - outputHtml = outputHtml + ''; - } - outputHtml = outputHtml + '
    scene_startscene_endscene_totalaudio_lendescription' + currDesc.start_time +'' + currDesc.end_time +'' + currDesc.total_time +'' + currDesc.audio_length +'' + currDesc.description +'' + currDesc.ocr + '
    ' - $('.col-7').append(outputHtml); - }); + let targetUrl = `/getDescriptionData?videoId=${videoId}&volunteerId=${volunteerId}&aiType=${aiType}`; + $.get(targetUrl, function(response) { + let descriptionArr = response; + let outputHtml = + ''; + for (let i = 0; i < descriptionArr.length; i++) { + let currDesc = descriptionArr[i]; + let sceneNum = currDesc.scene_num; + let audioPath = currDesc.audiopath; + currDesc.audio = new Audio(audioPath); + let sceneStart = currDesc.start_time; + startTimesArr.push(sceneStart); + sceneMap.set(sceneNum, currDesc); + outputHtml = outputHtml + ""; + outputHtml = + outputHtml + + "" + + "" + + "" + + "" + + "" + + ""; + outputHtml = outputHtml + ""; + } + outputHtml = outputHtml + "
    scene_startscene_endscene_totalaudio_lendescription
    " + + currDesc.start_time + + "" + + currDesc.end_time + + "" + + currDesc.total_time + + "" + + currDesc.audio_length + + "" + + currDesc.description + + "" + + currDesc.ocr + + "
    "; + $(".col-7").append(outputHtml); + }); } - function onPlayerStateChange(event) { - // -1 – unstarted - // 0 – ended - // 1 – playing - // 2 – paused - // 3 – buffering - // 5 – video cued - let playerState = event.data; - switch(playerState) { - case YT.PlayerState.PLAYING: - console.log('i was called: playing state') - $("#btn-play-pause").html(''); - let scene = getSceneFromTime(videotime, startTimesArr); - currAudio = scene.audio; - if((player.getCurrentTime() - scene.start_time) < scene.audio_length) { - currAudio.currentTime = player.getCurrentTime() - scene.start_time; - currAudio.play(); - } - break; - case YT.PlayerState.PAUSED: - $("#btn-play-pause").html(''); - console.log('i was called: paused state') - if(currAudio) { - currAudio.pause(); - } - break; - case YT.PlayerState.UNSTARTED: - case YT.PlayerState.ENDED: - case YT.PlayerState.CUED: - $("#btn-play-pause").html(''); - console.log('i was called: cued state') - break; - } + // -1 – unstarted + // 0 – ended + // 1 – playing + // 2 – paused + // 3 – buffering + // 5 – video cued + let playerState = event.data; + switch (playerState) { + case YT.PlayerState.PLAYING: + console.log("i was called: playing state"); + $("#btn-play-pause").html( + '' + ); + let scene = getSceneFromTime(videotime, startTimesArr); + currAudio = scene.audio; + if (player.getCurrentTime() - scene.start_time < scene.audio_length) { + currAudio.currentTime = player.getCurrentTime() - scene.start_time; + currAudio.play(); + } + break; + case YT.PlayerState.PAUSED: + $("#btn-play-pause").html( + '' + ); + console.log("i was called: paused state"); + if (currAudio) { + currAudio.pause(); + } + break; + case YT.PlayerState.UNSTARTED: + case YT.PlayerState.ENDED: + case YT.PlayerState.CUED: + $("#btn-play-pause").html( + '' + ); + console.log("i was called: cued state"); + break; + } } //Returns index of array item whose value is equal-to or lesser-than(also nearest-in-value) the target (binary search algorithm) function getIndexOfInputArrForTargetVal(target, inputArr) { - let start = 0; - let end = inputArr.length - 1; - let index = -1; - while(start <= end) { - let mid = Math.floor((start + end)/2); - if(inputArr[mid] == target) { - index = mid; - break; - } else if(inputArr[mid] < target) { - index = mid; - start = mid + 1; - } else { - end = mid - 1; - } + let start = 0; + let end = inputArr.length - 1; + let index = -1; + while (start <= end) { + let mid = Math.floor((start + end) / 2); + if (inputArr[mid] == target) { + index = mid; + break; + } else if (inputArr[mid] < target) { + index = mid; + start = mid + 1; + } else { + end = mid - 1; } - return index; + } + return index; } -function isPlaying(audioElement) { - if(!audioElement) { - return false; - } - return !audioElement.paused; +function isPlaying(audioElement) { + if (!audioElement) { + return false; + } + return !audioElement.paused; } diff --git a/public/javascripts/speechtotext.js b/public/javascripts/speechtotext.js new file mode 100644 index 0000000..cfbea0e --- /dev/null +++ b/public/javascripts/speechtotext.js @@ -0,0 +1,88 @@ +let recordButton = document.getElementById("record-button"); +let questionTimestamp = -1; + +window.SpeechRecognition = + window.webkitSpeechRecognition || window.SpeechRecognition; +if ("SpeechRecognition" in window) { + // supportMsg.innerHTML = 'Your browser supports speech recognition.'; // speech recognition API supported + // alert("Speech supported"); +} else { + alert("Speech not supported"); +} + +const recognition = new window.SpeechRecognition(); +recognition.lang = "en-US"; +recognition.interimResults = false; + +async function askQuestion() { + console.log("Record button clicked..Recording...."); + player.pauseVideo(); + beep(); + questionTimestamp = videotime.toFixed(3); + recognition.start(); +} + +function fetchAnswer() { + console.log("Function to stop speech recognition"); +} + +recognition.onstart = function() { + console.log("Speech recognition service has started"); +}; + +recognition.onresult = event => { + const speechToText = event.results[0][0].transcript; + console.log("Recording stpped!"); + if (speechToText.length > 0) { + audio_clip.src = '../audio/earcons/ting.wav'; + let data = { + userId: volunteerId, + videoId: youtubeId, + question: speechToText, + youtubeTimestamp: questionTimestamp, + conditionType + }; + + $.post("/userStudy4/addQuestion", data, function(res) { + console.log("Question Added"); + console.log(res); + question_id = res.questionId; + noOfQuestionsAsked += 1; + console.log({ speechToText }); + fetchKeyFrames(speechToText); + // console.log('Confidence: ' + event.results[0][0].confidence); + }); + } else { + console.log({ speechToText }); + } +}; + +let button = document.getElementById("speak"); + +function speak(text) { + // Create a new instance of SpeechSynthesisUtterance. + + var msg = new SpeechSynthesisUtterance("Test"); + msg.text = text; + msg.volume = 1; + msg.rate = 1; + msg.pitch = 1; + msg.voice = speechSynthesis.getVoices().filter(function(voice) { + return voice.name == "Alex"; + })[0]; + window.speechSynthesis.speak(msg); + console.log("speech synthesis called:", text); +} + +function beep() { + let snd = new Audio( + "data:audio/wav;base64,//uQRAAAAWMSLwUIYAAsYkXgoQwAEaYLWfkWgAI0wWs/ItAAAGDgYtAgAyN+QWaAAihwMWm4G8QQRDiMcCBcH3Cc+CDv/7xA4Tvh9Rz/y8QADBwMWgQAZG/ILNAARQ4GLTcDeIIIhxGOBAuD7hOfBB3/94gcJ3w+o5/5eIAIAAAVwWgQAVQ2ORaIQwEMAJiDg95G4nQL7mQVWI6GwRcfsZAcsKkJvxgxEjzFUgfHoSQ9Qq7KNwqHwuB13MA4a1q/DmBrHgPcmjiGoh//EwC5nGPEmS4RcfkVKOhJf+WOgoxJclFz3kgn//dBA+ya1GhurNn8zb//9NNutNuhz31f////9vt///z+IdAEAAAK4LQIAKobHItEIYCGAExBwe8jcToF9zIKrEdDYIuP2MgOWFSE34wYiR5iqQPj0JIeoVdlG4VD4XA67mAcNa1fhzA1jwHuTRxDUQ//iYBczjHiTJcIuPyKlHQkv/LHQUYkuSi57yQT//uggfZNajQ3Vmz+Zt//+mm3Wm3Q576v////+32///5/EOgAAADVghQAAAAA//uQZAUAB1WI0PZugAAAAAoQwAAAEk3nRd2qAAAAACiDgAAAAAAABCqEEQRLCgwpBGMlJkIz8jKhGvj4k6jzRnqasNKIeoh5gI7BJaC1A1AoNBjJgbyApVS4IDlZgDU5WUAxEKDNmmALHzZp0Fkz1FMTmGFl1FMEyodIavcCAUHDWrKAIA4aa2oCgILEBupZgHvAhEBcZ6joQBxS76AgccrFlczBvKLC0QI2cBoCFvfTDAo7eoOQInqDPBtvrDEZBNYN5xwNwxQRfw8ZQ5wQVLvO8OYU+mHvFLlDh05Mdg7BT6YrRPpCBznMB2r//xKJjyyOh+cImr2/4doscwD6neZjuZR4AgAABYAAAABy1xcdQtxYBYYZdifkUDgzzXaXn98Z0oi9ILU5mBjFANmRwlVJ3/6jYDAmxaiDG3/6xjQQCCKkRb/6kg/wW+kSJ5//rLobkLSiKmqP/0ikJuDaSaSf/6JiLYLEYnW/+kXg1WRVJL/9EmQ1YZIsv/6Qzwy5qk7/+tEU0nkls3/zIUMPKNX/6yZLf+kFgAfgGyLFAUwY//uQZAUABcd5UiNPVXAAAApAAAAAE0VZQKw9ISAAACgAAAAAVQIygIElVrFkBS+Jhi+EAuu+lKAkYUEIsmEAEoMeDmCETMvfSHTGkF5RWH7kz/ESHWPAq/kcCRhqBtMdokPdM7vil7RG98A2sc7zO6ZvTdM7pmOUAZTnJW+NXxqmd41dqJ6mLTXxrPpnV8avaIf5SvL7pndPvPpndJR9Kuu8fePvuiuhorgWjp7Mf/PRjxcFCPDkW31srioCExivv9lcwKEaHsf/7ow2Fl1T/9RkXgEhYElAoCLFtMArxwivDJJ+bR1HTKJdlEoTELCIqgEwVGSQ+hIm0NbK8WXcTEI0UPoa2NbG4y2K00JEWbZavJXkYaqo9CRHS55FcZTjKEk3NKoCYUnSQ0rWxrZbFKbKIhOKPZe1cJKzZSaQrIyULHDZmV5K4xySsDRKWOruanGtjLJXFEmwaIbDLX0hIPBUQPVFVkQkDoUNfSoDgQGKPekoxeGzA4DUvnn4bxzcZrtJyipKfPNy5w+9lnXwgqsiyHNeSVpemw4bWb9psYeq//uQZBoABQt4yMVxYAIAAAkQoAAAHvYpL5m6AAgAACXDAAAAD59jblTirQe9upFsmZbpMudy7Lz1X1DYsxOOSWpfPqNX2WqktK0DMvuGwlbNj44TleLPQ+Gsfb+GOWOKJoIrWb3cIMeeON6lz2umTqMXV8Mj30yWPpjoSa9ujK8SyeJP5y5mOW1D6hvLepeveEAEDo0mgCRClOEgANv3B9a6fikgUSu/DmAMATrGx7nng5p5iimPNZsfQLYB2sDLIkzRKZOHGAaUyDcpFBSLG9MCQALgAIgQs2YunOszLSAyQYPVC2YdGGeHD2dTdJk1pAHGAWDjnkcLKFymS3RQZTInzySoBwMG0QueC3gMsCEYxUqlrcxK6k1LQQcsmyYeQPdC2YfuGPASCBkcVMQQqpVJshui1tkXQJQV0OXGAZMXSOEEBRirXbVRQW7ugq7IM7rPWSZyDlM3IuNEkxzCOJ0ny2ThNkyRai1b6ev//3dzNGzNb//4uAvHT5sURcZCFcuKLhOFs8mLAAEAt4UWAAIABAAAAAB4qbHo0tIjVkUU//uQZAwABfSFz3ZqQAAAAAngwAAAE1HjMp2qAAAAACZDgAAAD5UkTE1UgZEUExqYynN1qZvqIOREEFmBcJQkwdxiFtw0qEOkGYfRDifBui9MQg4QAHAqWtAWHoCxu1Yf4VfWLPIM2mHDFsbQEVGwyqQoQcwnfHeIkNt9YnkiaS1oizycqJrx4KOQjahZxWbcZgztj2c49nKmkId44S71j0c8eV9yDK6uPRzx5X18eDvjvQ6yKo9ZSS6l//8elePK/Lf//IInrOF/FvDoADYAGBMGb7FtErm5MXMlmPAJQVgWta7Zx2go+8xJ0UiCb8LHHdftWyLJE0QIAIsI+UbXu67dZMjmgDGCGl1H+vpF4NSDckSIkk7Vd+sxEhBQMRU8j/12UIRhzSaUdQ+rQU5kGeFxm+hb1oh6pWWmv3uvmReDl0UnvtapVaIzo1jZbf/pD6ElLqSX+rUmOQNpJFa/r+sa4e/pBlAABoAAAAA3CUgShLdGIxsY7AUABPRrgCABdDuQ5GC7DqPQCgbbJUAoRSUj+NIEig0YfyWUho1VBBBA//uQZB4ABZx5zfMakeAAAAmwAAAAF5F3P0w9GtAAACfAAAAAwLhMDmAYWMgVEG1U0FIGCBgXBXAtfMH10000EEEEEECUBYln03TTTdNBDZopopYvrTTdNa325mImNg3TTPV9q3pmY0xoO6bv3r00y+IDGid/9aaaZTGMuj9mpu9Mpio1dXrr5HERTZSmqU36A3CumzN/9Robv/Xx4v9ijkSRSNLQhAWumap82WRSBUqXStV/YcS+XVLnSS+WLDroqArFkMEsAS+eWmrUzrO0oEmE40RlMZ5+ODIkAyKAGUwZ3mVKmcamcJnMW26MRPgUw6j+LkhyHGVGYjSUUKNpuJUQoOIAyDvEyG8S5yfK6dhZc0Tx1KI/gviKL6qvvFs1+bWtaz58uUNnryq6kt5RzOCkPWlVqVX2a/EEBUdU1KrXLf40GoiiFXK///qpoiDXrOgqDR38JB0bw7SoL+ZB9o1RCkQjQ2CBYZKd/+VJxZRRZlqSkKiws0WFxUyCwsKiMy7hUVFhIaCrNQsKkTIsLivwKKigsj8XYlwt/WKi2N4d//uQRCSAAjURNIHpMZBGYiaQPSYyAAABLAAAAAAAACWAAAAApUF/Mg+0aohSIRobBAsMlO//Kk4soosy1JSFRYWaLC4qZBYWFRGZdwqKiwkNBVmoWFSJkWFxX4FFRQWR+LsS4W/rFRb/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////VEFHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAU291bmRib3kuZGUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMjAwNGh0dHA6Ly93d3cuc291bmRib3kuZGUAAAAAAAAAACU=" + ); + snd.play(); +} + +async function doubleBeep() { + let snd = new Audio("../audio/earcons/beep-09.wav"); + + snd.play(); +} diff --git a/public/javascripts/tutorialSteps.js b/public/javascripts/tutorialSteps.js new file mode 100644 index 0000000..4d1cbee --- /dev/null +++ b/public/javascripts/tutorialSteps.js @@ -0,0 +1,191 @@ +let isFirstStep1 = true; +let isFirstStep2 = true; +let isFirstStep3 = true; +let isFirstStep4 = true; +let isFirstStep5 = true; + +class Step { + constructor(init, recurringSpeech, action, check) { + this.init = init; + this.recurringSpeech = recurringSpeech; + this.action = action; + this.check = check; + } +} + +let tutorialSteps = []; + +let step1 = new Step( + function() { + this.init = null; + }, + function() { + if (isFirstStep1) { + audio_clip.src = "../audio/tutorial_1_1.mp3"; + audio_clip.onended = function() { + audio_clip.src = "../audio/tutorial_1_2.mp3"; + audio_clip.onended = function() { + audio_clip.src = null; + }; + }; + isFirstStep1 = false; + } else { + audio_clip.src = "../audio/tutorial_1_2.mp3"; + audio_clip.onended = function() { + audio_clip.src = null; + }; + } + }, + function() { + if (player.getPlayerState == YT.PlayerState.PLAYING) pauseVideo(); + }, + function() { + if (playPause >= 1) { + return true; + } else { + return false; + } + } +); + +let step2 = new Step( + function() { + this.init = null; + this.count = 0; + }, + + function() { + if (isFirstStep2) { + audio_clip.src = "../audio/tutorial_2_1.mp3"; + audio_clip.onended = function() { + audio_clip.src = "../audio/tutorial_2_2.mp3"; + audio_clip.onended = function() { + audio_clip.src = null; + }; + }; + isFirstStep2 = false; + } else { + audio_clip.src = "../audio/tutorial_2_2.mp3"; + audio_clip.onended = function() { + audio_clip.src = null; + }; + } + }, + function() { + if ((player.getPlayerState = YT.PlayerState.PLAYING)) pauseVideo(); + }, + function() { + if (this.count < 50) { + this.count += 1; + return false; + } else { + pauseVideo(); + return true; + } + } +); + +let step3 = new Step( + function() { + console.log("tutorial 3 1"); + audio_clip.src = "../audio/tutorial_3_1.mp3"; + audio_clip.onended = function() { + audio_clip.src = null; + }; + this.init = null; + }, + + function() { + if (isFirstStep3) { + audio_clip.src = "../audio/tutorial_3_1.mp3"; + audio_clip.onended = function() { + audio_clip.src = "../audio/tutorial_3_2.mp3"; + audio_clip.onended = function() { + audio_clip.src = null; + }; + }; + isFirstStep3 = false; + } else { + audio_clip.src = "../audio/tutorial_3_2.mp3"; + audio_clip.onended = function() { + audio_clip.src = null; + }; + } + }, + function() { + if ((player.getPlayerState = YT.PlayerState.PLAYING)) pauseVideo(); + }, + function() { + if (captionsRequested >= 1) { + return true; + } + } +); + +let step4 = new Step( + function() { + audio_clip.src = "../audio/tutorial_4_1.mp3"; + audio_clip.onended = function() { + audio_clip.src = null; + }; + this.init = null; + }, + + function() { + if (isFirstStep4) { + audio_clip.src = "../audio/tutorial_4_1.mp3"; + audio_clip.onended = function() { + audio_clip.src = "../audio/tutorial_4_2.mp3"; + audio_clip.onended = function() { + audio_clip.src = null; + }; + }; + isFirstStep4 = false; + } else { + audio_clip.src = "../audio/tutorial_4_2.mp3"; + audio_clip.onended = function() { + audio_clip.src = null; + }; + } + }, + function() { + if ((player.getPlayerState = YT.PlayerState.PLAYING)) pauseVideo(); + }, + function() { + if (questionsAsked >= 1) { + return true; + } else { + return false; + } + } +); + +let step5 = new Step( + function() { + this.init = null; + }, + + function() { + if (isFirstStep4) { + audio_clip.src = "../audio/tutorial_5_1.mp3"; + audio_clip.onended = function() { + audio_clip.src = "../audio/tutorial_5_2.mp3"; + audio_clip.onended = function() { + audio_clip.src = null; + }; + }; + isFirstStep4 = false; + } else { + audio_clip.src = "../audio/tutorial_5_2.mp3"; + audio_clip.onended = function() { + audio_clip.src = null; + }; + } + }, + function() { + if ((player.getPlayerState = YT.PlayerState.PLAYING)) pauseVideo(); + }, + function() {} +); + +tutorialSteps.push(step1, step2, step3, step4, step5); diff --git a/public/javascripts/tutorialSteps.js.bup b/public/javascripts/tutorialSteps.js.bup new file mode 100644 index 0000000..897230a --- /dev/null +++ b/public/javascripts/tutorialSteps.js.bup @@ -0,0 +1,301 @@ +/******************** Original File ********************/ +class Step { + constructor(init, recurringSpeech, action, check) { + this.init = init; + this.recurringSpeech = recurringSpeech; + this.action = action; + this.check = check; + } +} + +let tutorialSteps = []; + +let step1 = new Step( + function() { + //tutorial_1_1.mp3 + // audio_clip.src = "../audio/tutorial_1_1.mp3"; + // audio_clip.onended = function() { + // audio_clip.src = null; + // } + speak( + "If you are unsure what to do at any point, press the I key to hear a reminder on the current task. Let's start. Pressing the spacebar will control video playback. Try this now" + ); + this.init = null; + }, + function() { + + // audio_clip.src = "../audio/tutorial_1_2.mp3"; + // audio_clip.onended = function() { + // audio_clip.src = null; + // } + // //tutorial_1_2.mp3 + speak("Press the space key twice to play and pause the video"); + }, + function() { + if ((player.getPlayerState = YT.PlayerState.PLAYING)) pauseVideo(); + }, + function() { + if (playPause >= 1) { + return true; + } else { + return false; + } + } +); + +let step2 = new Step( + function() { + //tutorial_2_1.mp3 + // let randomNum = Math.random(); + // audio_clip.src = "../audio/tutorial_2_1.mp3"; + // audio_clip.onended = function() { + // audio_clip.src = null; + // } + speak( + "Great. While the video plays, you will hear descriptions whenever there is text on the screen or a scene change. The video will continue playing during a description, or pause during an extended description. You can get additional information by requesting a description or by asking a question. First, let's try requesting a description. " + ); + this.init = null; + }, + + function() { + //tutorial_2_2.mp3 + // audio_clip.src = "../audio/tutorial_2_2.mp3"; + // audio_clip.onended = function() { + // audio_clip.src = null; + // } + speak( + "Press the spacebar to continue and then press D at any time to get a caption of the current image." + ); + }, + function() { + if ((player.getPlayerState = YT.PlayerState.PLAYING)) pauseVideo(); + }, + function() { + if (captionsRequested >= 1) { + return true; + } + } +); + +let step3 = new Step( + function() { + //tutorial_3_1.mp3 + // audio_clip.src = "../audio/tutorial_3_1.mp3"; + // audio_clip.onended = function() { + // audio_clip.src = null; + // } + speak( + "Good. Now, try asking a question about the current frame. You can ask something like, how many people are in the scene?" + ); + this.init = null; + }, + + function() { + //tutorial_3_2.mp3 + // audio_clip.src = "../audio/tutorial_3_2.mp3"; + // audio_clip.onended = function() { + // audio_clip.src = null; + // } + speak( + "Press the Q key to ask a question. After the beep, ask your question. " + ); + }, + function() { + if ((player.getPlayerState = YT.PlayerState.PLAYING)) pauseVideo(); + }, + function() { + if (questionsAsked >= 1) { + return true; + } else { + return false; + } + } +); + +let step4 = new Step( + //tutorial_4_1.mp3 + function() { + // audio_clip.src = "../audio/tutorial_4_1.mp3"; + // audio_clip.onended = function() { + // audio_clip.src = null; + // } + speak( + "Great Question. Continue watching the video and request more information to get familiar with the interface. At the end of the video, a button will appear to take you back to the home page." + ); + this.init = null; + }, + + function() { + //tutorial_4_2.mp3 + // audio_clip.src = "../audio/tutorial_4_2.mp3"; + // audio_clip.onended = function() { + // audio_clip.src = null; + // } + speak( + "Remember to press the Q key to ask a question, or the D key to get a description" + ); + }, + function() { + if ((player.getPlayerState = YT.PlayerState.PLAYING)) pauseVideo(); + }, + function() { + return false; + } +); + +tutorialSteps.push(step1, step2, step3, step4); + + + + +/********************************************************/ + + + + + +class Step { + constructor(init, recurringSpeech, action, check) { + this.init = init; + this.recurringSpeech = recurringSpeech; + this.action = action; + this.check = check; + } +} + +let tutorialSteps = []; + +let step1 = new Step( + function() { + //tutorial_1_1.mp3 + audio_clip.src = "../audio/tutorial_1_1.mp3"; + audio_clip.onended = function() { + audio_clip.src = null; + } + // speak( + // "If you are unsure what to do at any point, press the I key to hear a reminder on the current task. Let's start. Pressing the spacebar will control video playback. Try this now" + // ); + this.init = null; + }, + function() { + + audio_clip.src = "../audio/tutorial_1_2.mp3"; + audio_clip.onended = function() { + audio_clip.src = null; + } + // //tutorial_1_2.mp3 + // speak("Press the space key twice to play and pause the video"); + // }, + function() { + if ((player.getPlayerState = YT.PlayerState.PLAYING)) pauseVideo(); + }, + function() { + if (playPause >= 1) { + return true; + } else { + return false; + } + } +); + +let step2 = new Step( + function() { + //tutorial_2_1.mp3 + // let randomNum = Math.random(); + audio_clip.src = "../audio/tutorial_2_1.mp3"; + audio_clip.onended = function() { + audio_clip.src = null; + } + // speak( + // "Great. While the video plays, you will hear descriptions whenever there is text on the screen or a scene change. The video will continue playing during a description, or pause during an extended description. You can get additional information by requesting a description or by asking a question. First, let's try requesting a description. " + // ); + this.init = null; + }, + + function() { + //tutorial_2_2.mp3 + audio_clip.src = "../audio/tutorial_2_2.mp3"; + audio_clip.onended = function() { + audio_clip.src = null; + } + // speak( + // "Press the spacebar to continue and then press D at any time to get a caption of the current image." + // ); + }, + function() { + if ((player.getPlayerState = YT.PlayerState.PLAYING)) pauseVideo(); + }, + function() { + if (captionsRequested >= 1) { + return true; + } + } +); + +let step3 = new Step( + function() { + //tutorial_3_1.mp3 + audio_clip.src = "../audio/tutorial_3_1.mp3"; + audio_clip.onended = function() { + audio_clip.src = null; + } + // speak( + // "Good. Now, try asking a question about the current frame. You can ask something like, how many people are in the scene?" + // ); + this.init = null; + }, + + function() { + //tutorial_3_2.mp3 + audio_clip.src = "../audio/tutorial_3_2.mp3"; + audio_clip.onended = function() { + audio_clip.src = null; + } + // speak( + // "Press the Q key to ask a question. After the beep, ask your question. " + // ); + }, + function() { + if ((player.getPlayerState = YT.PlayerState.PLAYING)) pauseVideo(); + }, + function() { + if (questionsAsked >= 1) { + return true; + } else { + return false; + } + } +); + +let step4 = new Step( + //tutorial_4_1.mp3 + function() { + audio_clip.src = "../audio/tutorial_4_1.mp3"; + audio_clip.onended = function() { + audio_clip.src = null; + } + // speak( + // "Great Question. Continue watching the video and request more information to get familiar with the interface. At the end of the video, a button will appear to take you back to the home page." + // ); + this.init = null; + }, + + function() { + //tutorial_4_2.mp3 + audio_clip.src = "../audio/tutorial_4_2.mp3"; + audio_clip.onended = function() { + audio_clip.src = null; + } + // speak( + // "Remember to press the Q key to ask a question, or the D key to get a description" + // ); + }, + function() { + if ((player.getPlayerState = YT.PlayerState.PLAYING)) pauseVideo(); + }, + function() { + return false; + } +); + +tutorialSteps.push(step1, step2, step3, step4); diff --git a/public/javascripts/uploadVideo.js b/public/javascripts/uploadVideo.js new file mode 100644 index 0000000..87e0a70 --- /dev/null +++ b/public/javascripts/uploadVideo.js @@ -0,0 +1,352 @@ +let accountId = '5a8c6ddf-900f-48b4-8a3a-0cd5cf48cb4b'; +let apiKey = 'dbbd2d92a19749759576466ed9cd61fa'; + + +$(document).ready(function() { + + $("#videoInsightForm").submit(function (e) { + e.preventDefault(); + let videoId = document.getElementById('videoId').value; + getVideoInsight(videoId); + + }); + + $("#btnInsertVideo").click(function (e) { + e.preventDefault(); + console.log('insert'); + let youtubeId = document.getElementById('youtubeId').value; + let title = document.getElementById('videoName').value; + let inputJson = JSON.parse(document.getElementById('insight').value); + let jsonBody = { + "youtubeId": youtubeId, + "title": title, + "inputJson": inputJson + }; + console.log({jsonBody}); + $.ajax({ + type: 'POST', + timeout: 1*60*60*1000, + url : "/addvideo", + dataType: "json", + async:true, + crossDomain:true, + data: jsonBody, + success: function (response) { + window.alert(JSON.stringify(response)); + }, + error: function (response, textStatus, errorThrown) { + console.log(errorThrown); + window.alert(JSON.stringify(response)); + } + }); + }); + + + $("#btnDownload").click(function (e) { + e.preventDefault(); + console.log('download clicked'); + let url = document.getElementById('url').value; + let videoName = document.getElementById('videoName').value; + // http://www.youtube.com/watch?v=90AiXO1pAiA + console.log({url}); + videoURL = '/vi/downloadVideo?videoUrl='+ url+ '&title='+videoName; + + $.ajax({ + type: 'GET', + url : videoURL, + success: function (response) { + $('#alertDownload').html('
    ×'+ 'Success! File downloaded successfully' +'
    '); + console.log(response); + }, + error: function (response, textStatus, errorThrown) { + $('#alertDownload').html('
    ×'+ 'Error! An error occured while downloading the file' +'
    ') + console.log(JSON.stringify(response)); + } + }); + }); + + + + $("#audioUploadForm").submit(function (e) { + console.log("audio upload called"); + e.preventDefault(); + $('#alertUploadAudio').html('
    ×'+ 'File uploading...' +'
    '); + + let data = new FormData(e.target); + $.ajax({ + type: 'POST', + enctype: 'multipart/form-data', + url : 'http://18.221.192.73:8081/v1/audioclips/startspeechtotext', + data, + processData: false, + contentType: false, + cache: false, + success: function (response) { + document.getElementById('orderid').value = response.result.orderId; + $('#alertUploadAudio').html('
    ×'+ 'Success! File uploaded successfully' +'
    '); + console.log(response); + }, + error: function (response, textStatus, errorThrown) { + let err = JSON.stringify(response); + $('#alertUploadAudio').html('
    ×'+ 'Error! ' + 'File not uploaded' +'
    '); + console.log(err); + } + }); + + }); + + $("#audioInsightForm").submit(function (e) { + console.log("audio insight called"); + e.preventDefault(); + let orderid = document.getElementById('orderid').value; + let url = 'http://18.221.192.73:8081/v1/audioclips/getspeechtotextresult?orderid=' + orderid; + + $.get(url, function(response) { + console.log(response); + let audioInsight = document.getElementById('audioInsight'); + audioInsight.value = JSON.stringify(response); + }); + + }); + + + $("#uploadVideoForm").submit(function (e) { + + + console.log("submit called"); + e.preventDefault(); + + + $('#alertUpload').html('
    ×'+ 'Please wait...' +'
    '); + let data = new FormData(e.target); + for(let entry of data.entries()){ + console.log(entry); + } + + let videoName = document.getElementById('videoName').value; + + console.log({data}); + let tokenUrl = `https://api.videoindexer.ai/Auth/trial/Accounts?generateAccessTokens=true&allowEdit=true`; + console.log({tokenUrl}); + $.ajax({ + headers: { 'Ocp-Apim-Subscription-Key': apiKey }, + url: tokenUrl, + dataType: 'json', + type: 'GET', + success: function (response) { + let accessToken = response[0].accessToken; + console.log('token->' + accessToken); + let url = `https://api.videoindexer.ai/trial/Accounts/${accountId}/Videos?name=${videoName}&privacy=Public&indexingPreset=Default&streamingPreset=Default&sendSuccessEmail=False&accessToken=${accessToken}`; + $.ajax({ + type: 'POST', + enctype: 'multipart/form-data', + url, + data, + processData: false, + contentType: false, + cache: false, + success: function (response) { + console.log(response); + let id = response.id; + document.getElementById('videoId').value = id; + $('#alertUpload').html('
    ×'+ 'Success! File Uploaded to Video Indexer' +'
    '); + }, + error: function (response, textStatus, errorThrown) { + let errMessage = JSON.stringify(response); + console.log(JSON.stringify(response)); + $('#alertDownload').html('
    ×'+ `Error! An error occured while uploading the file: ${errMessage}` +'
    ') + + } + }); + + }, + error: function (response, textStatus, errorThrown) { + console.log(errorThrown); + } + }); + + }); + + + $("#btnMerge").click(function (e) { + console.log('merge clicked'); + let videoId = document.getElementById('youtubeId').value; + let audioId = document.getElementById('orderid').value; + let url = '/merge/mergeAudioVideo?videoId=' + videoId + '&audioId=' + audioId; + let params = {audioId, videoId} + console.log('Inside merge:', params); + console.log({url}); + $.ajax({ + url, + type: 'GET', + success: function (response) { + window.alert('Scenes merged successfully'); + console.log({Success : response}); + }, + error: function (response, textStatus, errorThrown) { + console.log(errorThrown.data); + console.log({response}); + window.alert('Error: Merge unsuccessful'); + } + }); + + }); +}); + + + + +function getVideoInsight(videoId){ + let location1= 'trial'; + let url = `https://api.videoindexer.ai/${location1}/accounts/${accountId}/videos/${videoId}/Index/?language=en-US`; + console.log({url}); + $.ajax({ + url, + dataType: 'json', + type: 'GET', + success: function (response) { + let insight = JSON.stringify(response); + document.getElementById('insight').value = insight; + // console.log(insight); + }, + error: function (response, textStatus, errorThrown) { + console.log(errorThrown); + } + }); + +} + + +// function execute(videoId){ +// let audioData = document.getElementById('audioInsight').value; +// let audioInsight = this.parseAudioFile(audioData); +// console.log({audioInsight}); + +// let url = '/merge/sceneData?videoId=' + videoId; + +// $.ajax({ +// url, +// dataType: 'json', +// type: 'GET', +// success: function (response) { +// console.log({scene: response}); +// let videoInsight = []; +// let data = response; +// for(let i=0; i< data.length; i++){ +// data[i].merged = []; +// videoInsight.push(data[i]); +// } +// console.log({videoInsight}); +// let mergedList = merge(videoInsight, audioInsight); +// console.log({arr: mergedList}); + +// $.ajax({ +// url : '/merge/scene', +// type: 'POST', +// data : {mergedList}, +// datatype: 'json', +// success: function (response) { +// window.alert('Success: ', JSON.stringify(response)); +// }, +// error: function (response, textStatus, errorThrown) { +// window.alert('Error: Merge failed'); +// console.log(JSON.stringify(errorThrown)); +// } +// }); +// }, +// error: function (response, textStatus, errorThrown) { + +// console.log(errorThrown); +// response.send(errorThrown); +// } +// }); + + +// } + + +// function parseAudioFile(data){ +// let obj = JSON.parse(data); +// let sentence = obj.result.sentence; +// let arr =[]; +// for(let i=0; i< sentence.length; i++){ +// // let instance = scenes[i].instances[0]; +// let start = sentence[i].startTime; +// let end = sentence[i].endTime; +// let data = { +// start, +// end +// } +// arr.push(data); +// } + +// return arr; +// } + + + +// function merge(videoInsight, audioInsight){ +// let videoIndex = 0; +// let mergedList = []; +// mergedList.push(videoInsight[videoIndex++]); +// let i = 0; +// while(i < audioInsight.length && videoIndex < videoInsight.length){ +// let size = mergedList.length; +// let s1 = mergedList[size-1].start_time; +// let e1 = mergedList[size-1].end_time; +// let description = mergedList[size-1].original_description; +// let ocr = mergedList[size-1].original_ocr; + +// let s2 = audioInsight[i].start; +// let e2 = audioInsight[i].end; + +// if((s2 >= s1 && s2 <= e1) && (e2 >= s1 && e2 <= e1)) { +// i++; +// continue; +// } + +// if(Math.min(s2, e2) >= e1) { +// mergedList.push(videoInsight[videoIndex++]); +// continue; +// } + +// //merge condition +// if(s2 >= s1 && e2 > e1) { +// let sceneObj = mergedList[mergedList.length-1]; +// let next = videoInsight[videoIndex++]; +// let min = Math.min(s1, next.start_time); +// let max = Math.max(e1, next.end_time); +// sceneObj.start_time = min; +// sceneObj.end_time = max; +// sceneObj.original_description = description + next.original_description; +// sceneObj.original_ocr = ocr + next.original_ocr; + +// // let arr = sceneObj.merged; +// // arr.push(next.scene_id); +// sceneObj.merged.push(next.scene_id); +// mergedList.splice(mergedList.length-1); +// mergedList.push(sceneObj); +// }else if(s2 >= e1) { +// mergedList.push(videoInsight[videoIndex++]); +// } + +// i++; +// } + +// for(let j = videoIndex; j ${rows[i].name} ` + // generatedHtml += ` ${rows[i].name} ` + } + $( ".list-group" ).append(generatedHtml); + }); +} \ No newline at end of file diff --git a/public/javascripts/userStudy4/usrStudyVidList4.js b/public/javascripts/userStudy4/usrStudyVidList4.js new file mode 100644 index 0000000..6bf28e3 --- /dev/null +++ b/public/javascripts/userStudy4/usrStudyVidList4.js @@ -0,0 +1,132 @@ +let userId = ""; +let tutorialHtml = ""; +let isOnlyTutorial = true; + +$(document).ready(function() { + let url = location.search.substring(1); + userId = new URLSearchParams(url).get("userId"); + + fetchTutorialVideo(); +}); + +function fetchTutorialVideo() { + let targetUrl = "/userStudy4/tutorialVideo"; + $.get(targetUrl, function(vidArr) { + // let generatedHtml = ''; + let userName = vidArr[0].name; + for (let i = 0; i < vidArr.length; i++) { + let vid = vidArr[i]; + let videoId = vid.video_id; + let videoTitle = vid.title; + let volunteerId = vid.volunteer_id; + let isAiSupported = vid.ai_support; + let isOnDemandSupported = vid.on_demand_support; + let aiSupportStr = isAiSupported + ? "With baseline descriptions" + : "Without baseline descriptions"; + let onDemandSupportStr = isOnDemandSupported + ? "with InfoBot" + : "without InfoBot"; + let aiParam = vid.ai_support === true ? "yes" : "no"; + let surveyLink = "#"; + + tutorialHtml += ` + + Tutorial Video: ${aiSupportStr} and ${onDemandSupportStr}`; + + tutorialHtml += `
    `; + } + let targetUrl = "/userStudy4/users?userId=" + userId; + let checkTutorialUrl = "/userStudy4/checkTutorial?userId=" + userId; + $.get(checkTutorialUrl, function(res) { + console.log(res); + if (res[0]["is_tutorial_complete"]) { + fetchVideo(targetUrl); + } else { + $(".list-group").append(tutorialHtml); + } + }); + }); +} + +function fetchVideo(targetUrl) { + $.get(targetUrl, function(vidJsonArr) { + updateUI(vidJsonArr); + }); +} + +function updateUI(vidArr) { + console.log(vidArr); + // let generatedHtml = ''; + let generatedHtml = tutorialHtml; + //unused variable + // let userName = vidArr[0].name; + for (let i = 0; i < vidArr.length; i++) { + isOnlyTutorial = false; + let vid = vidArr[i]; + console.log(vid); + let videoId = vid.video_id; + let videoTitle = vid.title; + let volunteerId = vid.volunteer_id; // on-demand Describer + let isAiSupported = vid.ai_support; + let isOnDemandSupported = vid.on_demand_support; + let describerId = + volunteerId == "f03cb948-474b-4084-9192-650ba62d396b" ? 1 : 2; + let aiSupportStr = isAiSupported + ? "With baseline descriptions" + : "Without baseline descriptions"; + let onDemandSupportStr = isOnDemandSupported + ? "with InfoBot" + : "without InfoBot"; + let aiParam = vid.ai_support === true ? "yes" : "no"; + let surveyLink = "#"; + + console.log(vid); + if (isOnDemandSupported) { + if (isAiSupported) { + if (describerId == 1) { + surveyLink = `https://sfsu.co1.qualtrics.com/jfe/form/SV_1Gn9oBUjTKWWOVf?userId=${userId}&video=${videoId}`; //on demand, human baseline + } else { + surveyLink = `https://sfsu.co1.qualtrics.com/jfe/form/SV_8tX3yXA6hte1I3j?userId=${userId}&video=${videoId}`; //on demand, ai baseline + } + } else { + surveyLink = `https://sfsu.co1.qualtrics.com/jfe/form/SV_097bW81QeNaAAzb?userId=${userId}&video=${videoId}`; //chatbot only + } + generatedHtml += ` + + + Video ${i + 1}: ${aiSupportStr} and ${onDemandSupportStr}`; + } else { + if (isAiSupported) { + if (describerId == 2) { + console.log(aiSupportStr); + surveyLink = `https://sfsu.co1.qualtrics.com/jfe/form/SV_3vHQEYhWIdUAyb3?userId=${userId}&video=${videoId}`; //ai baseline + } else { + surveyLink = `https://sfsu.co1.qualtrics.com/jfe/form/SV_0qhAkLfjQ5cQCKV?userId=${userId}&video=${videoId}`; //human baseline + } + generatedHtml += ` + + + Video ${i + + 1}: ${aiSupportStr} and ${onDemandSupportStr}`; + } else { + surveyLink = `https://sfsu.co1.qualtrics.com/jfe/form/SV_2iBERH5Z4aqR5aZ?userId=${userId}&video=${videoId}`; //no tools + generatedHtml += ` + + + Video ${i + + 1}: ${aiSupportStr} and ${onDemandSupportStr}`; + } + } + generatedHtml += ` + + User Survey for Video ${i + + 1}
    `; + } + if(!isOnlyTutorial){ + generatedHtml += ` + + General Survey`; + } + $(".list-group").append(generatedHtml); +} diff --git a/public/javascripts/videoaudio.js b/public/javascripts/videoaudio.js index e4c5902..6e88be9 100644 --- a/public/javascripts/videoaudio.js +++ b/public/javascripts/videoaudio.js @@ -1,5 +1,5 @@ let player; -let youtubeId = ''; +let youtubeId = ""; let videotime = 0; let timeupdater = null; let allSceneArr = []; @@ -9,7 +9,7 @@ let sceneToBlockMap = new Map(); let userId; let prevSceneNum = -1; let timeAtWhichLastPaused = -1; -let previouslyHighlightedContainerId = '#doesnotexist'; +let previouslyHighlightedContainerId = "#doesnotexist"; let enableAi = false; let autoPauseAtEachScene = false; let timeElapsedOnPage = 0; @@ -18,524 +18,610 @@ let audioArr = []; let currAudio; $(document).ready(function() { - $('#btn-video-breakpoint-toggle').prop("checked", false); - userId = getUrlVars()["u"]; - enableAi = (getUrlVars()["ml"] == 'yes') ? true : false; - let href = location.href; - const lastPathSegment = new URL(href).pathname.split('/').pop(); - youtubeId = lastPathSegment; - fetchVideoInfo(youtubeId); - $("#btnScroll").click(function() { - resetVidModifiedTableData(); - }); - $('#btnSubmit').click(async function() { - let shoudShowAlert = true; - updateAll(shoudShowAlert); - }); - $('#btnTemp').click(async function() { - updateMainTables(); - }); - $('#btn-video-breakpoint-toggle').change(function() { - if($(this).is(":checked")) { - autoPauseAtEachScene = true; - } else { - autoPauseAtEachScene = false; - } - }); - $("#btn-play-pause").click(function() { - onPlayPauseButtonClick(); - }); - $('#btn-timer').click(function () { - isTimerPaused = !isTimerPaused; - if(isTimerPaused) { - $('#btn-timer').attr('src', '/images/play_white.png'); - } else { - $('#btn-timer').attr('src', '/images/pause_white.png'); - } - }); - setInterval(incrementSeconds, 1000); + $("#btn-video-breakpoint-toggle").prop("checked", false); + userId = getUrlVars()["u"]; + enableAi = getUrlVars()["ml"] == "yes" ? true : false; + let href = location.href; + const lastPathSegment = new URL(href).pathname.split("/").pop(); + youtubeId = lastPathSegment; + fetchVideoInfo(youtubeId); + $("#btnScroll").click(function() { + resetVidModifiedTableData(); + }); + $("#btnSubmit").click(async function() { + let shoudShowAlert = true; + updateAll(shoudShowAlert); + }); + $("#btnTemp").click(async function() { + updateMainTables(); + }); + $("#btn-video-breakpoint-toggle").change(function() { + if ($(this).is(":checked")) { + autoPauseAtEachScene = true; + } else { + autoPauseAtEachScene = false; + } + }); + $("#btn-play-pause").click(function() { + onPlayPauseButtonClick(); + }); + $("#btn-timer").click(function() { + isTimerPaused = !isTimerPaused; + if (isTimerPaused) { + $("#btn-timer").attr("src", "/images/play_white.png"); + } else { + $("#btn-timer").attr("src", "/images/pause_white.png"); + } + }); + setInterval(incrementSeconds, 1000); }); function incrementSeconds() { - if(!isTimerPaused) { - timeElapsedOnPage += 1; - $('#txt-timeElaspsed').html(formatTimeForTimer(timeElapsedOnPage)); - } + if (!isTimerPaused) { + timeElapsedOnPage += 1; + $("#txt-timeElaspsed").html(formatTimeForTimer(timeElapsedOnPage)); + } } function fetchVideoInfo(youtubeId) { - let targetUrl = '/fetchVideoData' + '?videoid=' + youtubeId; - $.get(targetUrl, function (response) { - let vidJson = response; - let vidTitle = vidJson.title; - let vidLength = vidJson.duration; - let blocksJsonArr = vidJson.info; - $('#txtVidtitle').html(vidTitle); - populateAllSceneArrAndAllBlockArr(blocksJsonArr); - sceneIdToListPositionMap = populateSceneIdToListPositionMap(blocksJsonArr); - for (var [key, value] of sceneIdToListPositionMap) { - let sceneId = key; - let sceneNum = sceneId.split('_')[2]; - let randomNum = Math.random(); - let descType = enableAi ? 'ai' : 'non_ai'; - let src = `../audio/${youtubeId}/${descType}/${userId}/${sceneNum}.mp3` + '?random='+randomNum; - let audioFile = new Audio(src); - audioArr.push(audioFile); - } - initializeDescriptionsUI(blocksJsonArr); - //setInterval(updateAll, 1000*60*2); - }); + let targetUrl = "/fetchVideoData" + "?videoid=" + youtubeId; + $.get(targetUrl, function(response) { + let vidJson = response; + let vidTitle = vidJson.title; + let vidLength = vidJson.duration; + let blocksJsonArr = vidJson.info; + $("#txtVidtitle").html(vidTitle); + populateAllSceneArrAndAllBlockArr(blocksJsonArr); + sceneIdToListPositionMap = populateSceneIdToListPositionMap(blocksJsonArr); + for (var [key, value] of sceneIdToListPositionMap) { + let sceneId = key; + let sceneNum = sceneId.split("_")[2]; + let randomNum = Math.random(); + let descType = enableAi ? "ai" : "non_ai"; + let src = + `../audio/${youtubeId}/${descType}/${userId}/${sceneNum}.mp3` + + "?random=" + + randomNum; + let audioFile = new Audio(src); + audioArr.push(audioFile); + } + initializeDescriptionsUI(blocksJsonArr); + //setInterval(updateAll, 1000*60*2); + }); } function initializeDescriptionsUI(blocksJsonArr) { - let outputHtml = ''; - for(let i=0; i
    • '; - for(let j=0; j
    ' - + '' - + '' - + '
    Scene ' + sceneNum + '' + scStartTimePretty + ' to ' + scFinishTimePretty + ' minutes
    Description
    ' - + '' - + '
    Text On-Screen
    ' - + '' - + '
    ' - + '
    '; - // if(j!=childSceneArr.length-1) { - // outputHtml = outputHtml + '' - // } - outputHtml = outputHtml - + '
    ' - + '
    '; - } - outputHtml = outputHtml - + ''; + let outputHtml = ""; + for (let i = 0; i < blocksJsonArr.length; i++) { + let currBlock = blocksJsonArr[i]; + let blockId = currBlock.block_id; + let blockNum = currBlock.block_num; + let blStartTime = currBlock.start_time; + let blFinishTime = currBlock.finish_time; + let baselineDesc = enableAi ? $.trim(currBlock.baseline_des) : ""; + let childSceneArr = currBlock.child_scenes; + outputHtml = + outputHtml + + '"; + for (let j = 0; j < childSceneArr.length; j++) { + let currScene = childSceneArr[j]; + let sceneId = currScene.sc_id; + let sceneNum = currScene.sc_num; + let scStartTime = currScene.start_time; + let scFinishTime = currScene.finish_time; + let sceneOcr = enableAi ? $.trim(currScene.original_ocr) : ""; + let sceneDesc = enableAi ? $.trim(currScene.original_description) : ""; + let scStartTimePretty = millisToMinutesAndSeconds(scStartTime); + let scFinishTimePretty = millisToMinutesAndSeconds(scFinishTime); + outputHtml = + outputHtml + + '
  • ' + + "' + + '" + + '
    Scene " + + sceneNum + + "" + + scStartTimePretty + + " to " + + scFinishTimePretty + + ' minutes
    Description
    ' + + '' + + '
    Text On-Screen
    ' + + '' + + '
    " + + '
    '; + // if(j!=childSceneArr.length-1) { + // outputHtml = outputHtml + '' + // } + outputHtml = outputHtml + "
    " + "
    • "; } - $('#ul-scene-list').html(outputHtml); - $('.btn-update-scene-description').click(function() { - let sceneId = this.dataset.sceneid; - let sceneDescTxtId = '#s_d_' + sceneId; - let modifiedDescription = $(sceneDescTxtId).val(); - sendUpdateSceneDescriptionPostReq(sceneId, modifiedDescription); - }); - $('.btn-update-scene-ocr').click(function() { - let sceneId = this.dataset.sceneid; - let sceneOcrTxtId = '#s_o_' + sceneId; - let modifiedOcr = $(sceneOcrTxtId).val(); - sendUpdateOcrPostReq(sceneId, modifiedOcr); - }); - $('.btn-update-block-description').click(function() { - let blockId = this.dataset.blockid; - let blockDescTxtId ='#b_d_' + blockId; - let modifiedDescription = $(blockDescTxtId).val(); - console.log(blockDescTxtId + ', ' +blockId, ', ' + modifiedDescription); - sendUpdateBlockDescriptionPostReq(blockId, modifiedDescription); - }); - $("textarea").each(function(){ - auto_grow(this); - }); + outputHtml = + outputHtml + + '"; + } + $("#ul-scene-list").html(outputHtml); + $(".btn-update-scene-description").click(function() { + let sceneId = this.dataset.sceneid; + let sceneDescTxtId = "#s_d_" + sceneId; + let modifiedDescription = $(sceneDescTxtId).val(); + sendUpdateSceneDescriptionPostReq(sceneId, modifiedDescription); + }); + $(".btn-update-scene-ocr").click(function() { + let sceneId = this.dataset.sceneid; + let sceneOcrTxtId = "#s_o_" + sceneId; + let modifiedOcr = $(sceneOcrTxtId).val(); + sendUpdateOcrPostReq(sceneId, modifiedOcr); + }); + $(".btn-update-block-description").click(function() { + let blockId = this.dataset.blockid; + let blockDescTxtId = "#b_d_" + blockId; + let modifiedDescription = $(blockDescTxtId).val(); + console.log(blockDescTxtId + ", " + blockId, ", " + modifiedDescription); + sendUpdateBlockDescriptionPostReq(blockId, modifiedDescription); + }); + $("textarea").each(function() { + auto_grow(this); + }); } function millisToMinutesAndSeconds(secs) { - var minutes = Math.floor(secs / 60); - var seconds = ((secs % 60) / 1).toFixed(1); - return minutes + ":" + seconds; -} + var minutes = Math.floor(secs / 60); + var seconds = ((secs % 60) / 1).toFixed(1); + return minutes + ":" + seconds; +} function sendUpdateSceneDescriptionPostReq(sceneId, modifiedDescription) { - const body = { - userId: userId, - sceneId: sceneId, - modifiedDescription: modifiedDescription, - }; - $.post( '/updateSceneDescription', body, function(response) { - window.alert(JSON.stringify(response)); - }); + const body = { + userId: userId, + sceneId: sceneId, + modifiedDescription: modifiedDescription + }; + $.post("/updateSceneDescription", body, function(response) { + window.alert(JSON.stringify(response)); + }); } function sendUpdateOcrPostReq(sceneId, modifiedOcr) { - const body = { - userId: userId, - sceneId: sceneId, - modifiedOcr: modifiedOcr, - }; - $.post( '/updateSceneOCR', body, function(response) { - window.alert(JSON.stringify(response)); - }); + const body = { + userId: userId, + sceneId: sceneId, + modifiedOcr: modifiedOcr + }; + $.post("/updateSceneOCR", body, function(response) { + window.alert(JSON.stringify(response)); + }); } function sendUpdateBlockDescriptionPostReq(blockId, modifiedDescription) { - const body = { - userId: userId, - blockId: blockId, - modifiedDescription: modifiedDescription, - }; - $.post( '/updateBlockDescription', body, function(response) { - window.alert(JSON.stringify(response)); - }); + const body = { + userId: userId, + blockId: blockId, + modifiedDescription: modifiedDescription + }; + $.post("/updateBlockDescription", body, function(response) { + window.alert(JSON.stringify(response)); + }); } - function scrollToPosition(position) { - $('#ul-scene-list').animate({ - scrollTop: $('#ul-scene-list li:nth-child(' + position + ')').position().top - $('#ul-scene-list li:first').position().top - }, 'slow'); - $(previouslyHighlightedContainerId).removeClass('highlight'); - $('#ul-scene-list li:nth-child(' + position + ')' + ' .scene-container').addClass('highlight'); - previouslyHighlightedContainerId = '#ul-scene-list li:nth-child(' + position + ')' + ' .scene-container'; + $("#ul-scene-list").animate( + { + scrollTop: + $("#ul-scene-list li:nth-child(" + position + ")").position().top - + $("#ul-scene-list li:first").position().top + }, + "slow" + ); + $(previouslyHighlightedContainerId).removeClass("highlight"); + $( + "#ul-scene-list li:nth-child(" + position + ")" + " .scene-container" + ).addClass("highlight"); + previouslyHighlightedContainerId = + "#ul-scene-list li:nth-child(" + position + ")" + " .scene-container"; } function resetVidModifiedTableData() { - let url = '/reset/' + youtubeId + '/' + userId; - $.get(url, function(response) { - window.alert(JSON.stringify(response)); - }); + let url = "/reset/" + youtubeId + "/" + userId; + $.get(url, function(response) { + window.alert(JSON.stringify(response)); + }); } function onYouTubeIframeAPIReady() { - player = new YT.Player('player', { - height: '450', - width: '800', - videoId: youtubeId, - playerVars: { - autoplay: 0, - playsinline: 1, - enablejsapi: 1, - cc_load_policy: 0, - controls: 1, - fs: 0, - iv_load_policy: 3, - modestbranding: 1, - rel: 0, - showinfo: 0, - wmode: 'opaque' - }, - events: { - 'onReady': onPlayerReady, - 'onStateChange': onPlayerStateChange - } - }); + player = new YT.Player("player", { + height: "270", + width: "480", + videoId: youtubeId, + playerVars: { + autoplay: 0, + playsinline: 1, + enablejsapi: 1, + cc_load_policy: 0, + controls: 0, + disablekb: 1, + fs: 0, + iv_load_policy: 3, + modestbranding: 1, + rel: 0, + showinfo: 0, + wmode: "opaque" + }, + events: { + onReady: onPlayerReady, + onStateChange: onPlayerStateChange + } + }); } function onPlayerReady(event) { - player.setVolume(50); - $('#total_time').text(formatTime(player.getDuration())); - timeupdater = setInterval(updateTime, 100); + player.setVolume(50); + $("#total_time").text(formatTime(player.getDuration())); + timeupdater = setInterval(updateTime, 100); + document.getElementById("player").tabIndex = -1; } function updateTime() { - var oldTime = videotime; - if(player && player.getCurrentTime) { - videotime = player.getCurrentTime(); - } - if(videotime !== oldTime) { - onProgress(videotime); - } - $('#elapsed_time').text(formatTime(player.getCurrentTime())); + var oldTime = videotime; + if (player && player.getCurrentTime) { + videotime = player.getCurrentTime(); + } + if (videotime !== oldTime) { + onProgress(videotime); + } + $("#elapsed_time").text(formatTime(player.getCurrentTime())); } // when the time changes, this will be called. function onProgress(currentTime) { - currentTime = currentTime.toFixed(3); - let currScene = getSceneFromTime(currentTime, allSceneArr); - let sceneId = currScene.sc_id; - let sceneNum = currScene.sc_num; - let sceneEndTime = currScene.finish_time; - if(prevSceneNum != sceneNum) { - if((currAudio !== null) && (sceneNum !==0) && (isPlaying(currAudio))) { - pauseVideo(); - prevSceneNum = sceneNum-1; + currentTime = currentTime.toFixed(3); + let currScene = getSceneFromTime(currentTime, allSceneArr); + let sceneId = currScene.sc_id; + let sceneNum = currScene.sc_num; + let sceneEndTime = currScene.finish_time; + if (prevSceneNum != sceneNum) { + if (currAudio !== null && sceneNum !== 0 && isPlaying(currAudio)) { + pauseVideo(); + prevSceneNum = sceneNum - 1; + } else { + let listPosition = sceneIdToListPositionMap.get(sceneId); + scrollToPosition(listPosition); + + currAudio = audioArr[sceneNum]; + currAudio.play(); + currAudio.onended = function() { + console.log("player state is: " + player.getPlayerState()); + if (player.getPlayerState() == YT.PlayerState.PAUSED) { + console.log("I was called"); + playVideo(); } else { - let listPosition = sceneIdToListPositionMap.get(sceneId); - scrollToPosition(listPosition); - - currAudio = audioArr[sceneNum]; - currAudio.play(); - currAudio.onended = function() { - console.log('player state is: ' + player.getPlayerState()); - if(player.getPlayerState() == YT.PlayerState.PAUSED) { - console.log('I was called'); - playVideo(); - } else { - prevSceneNum = sceneNum; - } - }; - prevSceneNum = sceneNum; + prevSceneNum = sceneNum; } + }; + prevSceneNum = sceneNum; } - //prevSceneNum = sceneNum; - if(timeAtWhichLastPaused > currentTime) { - timeAtWhichLastPaused = currentTime; - } - if((currAudio !== null) && (sceneNum !==0) && (isPlaying(currAudio)) && sceneEndTime - currentTime < 0.1 && currentTime - timeAtWhichLastPaused > 0.5 && autoPauseAtEachScene) { - pauseVideo(); - timeAtWhichLastPaused = currentTime; - //if(sceneToBlockMap.has(sceneId)) {//Check if it's the last scene of the current block - // let blockId = sceneToBlockMap.get(sceneId); - // let blockElementId = '#bl_cont_' + blockId; - } - let sceneElementId = '#sc_cont_' + sceneId; - $(previouslyHighlightedContainerId).removeClass('highlight'); - $(sceneElementId).addClass('highlight'); - previouslyHighlightedContainerId = sceneElementId; + } + //prevSceneNum = sceneNum; + if (timeAtWhichLastPaused > currentTime) { + timeAtWhichLastPaused = currentTime; + } + if ( + currAudio !== null && + sceneNum !== 0 && + isPlaying(currAudio) && + sceneEndTime - currentTime < 0.1 && + currentTime - timeAtWhichLastPaused > 0.5 && + autoPauseAtEachScene + ) { + pauseVideo(); + timeAtWhichLastPaused = currentTime; + //if(sceneToBlockMap.has(sceneId)) {//Check if it's the last scene of the current block + // let blockId = sceneToBlockMap.get(sceneId); + // let blockElementId = '#bl_cont_' + blockId; + } + let sceneElementId = "#sc_cont_" + sceneId; + $(previouslyHighlightedContainerId).removeClass("highlight"); + $(sceneElementId).addClass("highlight"); + previouslyHighlightedContainerId = sceneElementId; } function onPlayerStateChange(event) { - let playerState = event.data; - switch(playerState) { - case YT.PlayerState.PLAYING: - console.log('i was called: playing state') - $("#btn-play-pause").html(''); - break; - case YT.PlayerState.PAUSED: - $("#btn-play-pause").html(''); - timeAtWhichLastPaused = player.getCurrentTime(); - console.log(timeAtWhichLastPaused); - console.log('i was called: paused state') - break; - case YT.PlayerState.UNSTARTED: - case YT.PlayerState.ENDED: - case YT.PlayerState.CUED: - $("#btn-play-pause").html(''); - timeAtWhichLastPaused = player.getCurrentTime(); - console.log('i was called: cued state') - break; - } + let playerState = event.data; + switch (playerState) { + case YT.PlayerState.PLAYING: + console.log("i was called: playing state"); + $("#btn-play-pause").html( + '' + ); + break; + case YT.PlayerState.PAUSED: + $("#btn-play-pause").html( + '' + ); + timeAtWhichLastPaused = player.getCurrentTime(); + console.log(timeAtWhichLastPaused); + console.log("i was called: paused state"); + break; + case YT.PlayerState.UNSTARTED: + case YT.PlayerState.ENDED: + case YT.PlayerState.CUED: + $("#btn-play-pause").html( + '' + ); + timeAtWhichLastPaused = player.getCurrentTime(); + console.log("i was called: cued state"); + break; + } } function playVideo() { - player.playVideo(); + player.playVideo(); } function pauseVideo() { - player.pauseVideo(); + player.pauseVideo(); } function stopVideo() { - player.stopVideo(); + player.stopVideo(); } async function updateAll(shoudShowAlert) { - let postJson = fetchJsonBodyForAllTextFields(allSceneArr, allBlockArr, userId); - postJson.videoId = youtubeId; - // await $.post('/updateMultipleScenesAndBlocksForUser', postJson, function (response) { - await $.post('/saveAiDescription', postJson, function (response) { - if(shoudShowAlert) { - if(response.success) { - alert('Submitted Successfully\nYou will be redirected to the Video List Page'); - window.history.back(); - } else { - alert('Update Failed: ' + response.success); - } - } - }); + let postJson = fetchJsonBodyForAllTextFields( + allSceneArr, + allBlockArr, + userId + ); + postJson.videoId = youtubeId; + // await $.post('/updateMultipleScenesAndBlocksForUser', postJson, function (response) { + await $.post("/saveAiDescription", postJson, function(response) { + if (shoudShowAlert) { + if (response.success) { + alert( + "Submitted Successfully\nYou will be redirected to the Video List Page" + ); + window.history.back(); + } else { + alert("Update Failed: " + response.success); + } + } + }); } - async function updateMainTables() { - let postJson = fetchJsonBodyForAllTextFields2(allSceneArr, allBlockArr); - await $.post('/updateBlockAndScene', postJson, function (response) { - if(response.success) { - window.alert('Updated Successfully'); - } else { - window.alert('Update Failed'); - } - }); + let postJson = fetchJsonBodyForAllTextFields2(allSceneArr, allBlockArr); + await $.post("/updateBlockAndScene", postJson, function(response) { + if (response.success) { + window.alert("Updated Successfully"); + } else { + window.alert("Update Failed"); + } + }); } - function fetchJsonBodyForAllTextFields(allSceneArr, allBlockArr, userId) { - let parentJsonBody = {}; - let sceneArrJson = []; - let blockArrJson = []; - for(let i=0; i= 3) { + var formData = new FormData(); + formData.append("img_path", image); + formData.append("history", prev_history); + formData.append("csrfmiddlewaretoken", "{{ csrf_token }}"); + formData.append("socketid", ws_id); + formData.append("question", question); + formData.append("job_id", job_id); + formData.append("image_key", imageKey); + var URL = "https://dev-api.youdescribe.org/"; + // var URL = "https://cors-anywhere.herokuapp.com/http://13.59.47.124:8001/"; // to test from local + + $.ajax({ + type: "POST", + url: URL, + data: formData, + processData: false, + contentType: false + }).done(function(response) { + console.log(response); + }); + } +} + +function uuidv4() { + return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, function(c) { + var r = (Math.random() * 16) | 0, + v = c == "x" ? r : (r & 0x3) | 0x8; + return v.toString(16); + }); +} + +function updateQuestionInfo(imagePath, question_id){ + + let updateInfo = { + userId : volunteerId, + questionId : question_id, + keyframe : imagePath, + dialogId + } + + $.post("/userStudy4/updateQuestion", updateInfo, function(res) { + console.log("Question Updated"); + console.log(res); + }); +} + +async function checkQuestion() { + audio_clip.src = "/audio/tutorial_question.mp3"; + audio_clip.onended = function() { + // currAudio = null; + }; +} diff --git a/public/javascripts/visdialResponseFilter.js b/public/javascripts/visdialResponseFilter.js new file mode 100644 index 0000000..8e198f7 --- /dev/null +++ b/public/javascripts/visdialResponseFilter.js @@ -0,0 +1,35 @@ +let listOfWordsToFilter = new Set(["", "can\'t tell", "i can\'t tell"]); +let listOfSentencesToFilter = new Set(["can\'t tell", "i can\'t tell"]); + +async function filterResponse(input){ + let sentences = input.split('.'); + let output = ""; + + for(let i = 0; i < sentences.length; i++){ + + let sentence = sentences[i]; + let words = sentence.split(" "); + + if(listOfSentencesToFilter.has(sentence)){ + output += ("Chat bot is unable to answer. "); + continue; + } + + for(let j = 0; j < words.length; j++){ + + let word = words[j]; + if(listOfWordsToFilter.has(word)){ + output += ("Chat bot is unable to answer. "); + break; + } + if (j == words.length - 1){ + output += sentence; + output += ". "; + } + + } +} + + return output; +} + diff --git a/public/javascripts/withoutai.js b/public/javascripts/withoutai.js index 793e25b..48c7940 100644 --- a/public/javascripts/withoutai.js +++ b/public/javascripts/withoutai.js @@ -1,5 +1,5 @@ let player; -let youtubeId = ''; +let youtubeId = ""; let videotime = 0; let timeupdater = null; let allSceneArr = []; @@ -9,505 +9,579 @@ let sceneToBlockMap = new Map(); let userId; let prevSceneNum = -1; let timeAtWhichLastPaused = -1; -let previouslyHighlightedContainerId = '#doesnotexist'; +let previouslyHighlightedContainerId = "#doesnotexist"; let enableAi = false; let newTimerUpdater = null; let timeElapsedOnPage = 0; let isTimerPaused = false; $(document).ready(function() { - userId = getUrlVars()["u"]; - enableAi = (getUrlVars()["ai"] == 'yes') ? true : false; - let href = location.href; - const lastPathSegment = new URL(href).pathname.split('/').pop(); - youtubeId = lastPathSegment; - fetchVideoInfo(youtubeId); - $("#btnScroll").click(function() { - resetVidModifiedTableData(); - }); - $('#btnSubmit').click(async function() { - let shoudShowAlert = true; - updateAll(shoudShowAlert); - }); - $('#btnTemp').click(async function() { - updateMainTables(); - }); - $('.txt-scene-description').keyup(function() { - auto_grow(this); - }); - $('#btnAddScene').click(function() { - addSceneListItemToUi(); - }); - $('.sceneTimeEntry').timeEntry({ - spinnerImage: '../images/spinnerUpDown.png', - spinnerBigImage: '../images/spinnerUpDownBig.png', - spinnerSize: [15, 16, 0], - spinnerBigSize: [30, 32, 0], - unlimitedHours: true, - showSeconds: true, - spinnerIncDecOnly: true, - useMouseWheel: true - }); - $("#btn-play-pause").click(function() { - onPlayPauseButtonClick(); - }); - $('#btn-timer').click(function () { - isTimerPaused = !isTimerPaused; - if(isTimerPaused) { - $('#btn-timer').attr('src', '/images/play_white.png'); - } else { - $('#btn-timer').attr('src', '/images/pause_white.png'); - } - }); - setInterval(incrementSeconds, 1000); - $('#btnSubmit').click(async function() { - let sceneDataJson = fetchDataFromAllFields(); - let startArr = []; - for(let i=1; i< sceneDataJson.scenes.length; i++) { - let current = sceneDataJson.scenes[i]; - startArr.push(current.scene_time); - - } - console.log(startArr); - - let i = 0; - for(let key in sceneDataJson.scenes) { - let currScene = sceneDataJson.scenes[key]; - currScene.start_time = convertToSeconds(currScene.scene_time); - if(i === sceneDataJson.scenes.length-1){ - currScene.end_time = player.getDuration(); - }else{ - currScene.end_time = convertToSeconds(startArr[i++]); - } - currScene.modified_description = currScene.scene_desc; - currScene.modified_ocr = ''; - currScene.has_ai = false; - currScene.deletedScene = []; - currScene.videoId = youtubeId; - console.log(sceneDataJson.scenes[key]); - } - - $.post( '/saveNonAiDescription', sceneDataJson, function(response) { - if(response.success) { - alert('Submitted Successfully\nYou will be redirected to the Video List Page'); - window.history.back(); - } else { - alert('Update Failed: ' + response.success); - } - }); + userId = getUrlVars()["u"]; + enableAi = getUrlVars()["ai"] == "yes" ? true : false; + let href = location.href; + const lastPathSegment = new URL(href).pathname.split("/").pop(); + youtubeId = lastPathSegment; + fetchVideoInfo(youtubeId); + $("#btnScroll").click(function() { + resetVidModifiedTableData(); + }); + $("#btnSubmit").click(async function() { + let shoudShowAlert = true; + updateAll(shoudShowAlert); + }); + $("#btnTemp").click(async function() { + updateMainTables(); + }); + $(".txt-scene-description").keyup(function() { + auto_grow(this); + }); + $("#btnAddScene").click(function() { + addSceneListItemToUi(); + }); + $(".sceneTimeEntry").timeEntry({ + spinnerImage: "../images/spinnerUpDown.png", + spinnerBigImage: "../images/spinnerUpDownBig.png", + spinnerSize: [15, 16, 0], + spinnerBigSize: [30, 32, 0], + unlimitedHours: true, + showSeconds: true, + spinnerIncDecOnly: true, + useMouseWheel: true + }); + $("#btn-play-pause").click(function() { + onPlayPauseButtonClick(); + }); + $("#btn-timer").click(function() { + isTimerPaused = !isTimerPaused; + if (isTimerPaused) { + $("#btn-timer").attr("src", "/images/play_white.png"); + } else { + $("#btn-timer").attr("src", "/images/pause_white.png"); + } + }); + setInterval(incrementSeconds, 1000); + $("#btnSubmit").click(async function() { + let sceneDataJson = fetchDataFromAllFields(); + let startArr = []; + for (let i = 1; i < sceneDataJson.scenes.length; i++) { + let current = sceneDataJson.scenes[i]; + startArr.push(current.scene_time); + } + console.log(startArr); + + let i = 0; + for (let key in sceneDataJson.scenes) { + let currScene = sceneDataJson.scenes[key]; + currScene.start_time = convertToSeconds(currScene.scene_time); + if (i === sceneDataJson.scenes.length - 1) { + currScene.end_time = player.getDuration(); + } else { + currScene.end_time = convertToSeconds(startArr[i++]); + } + currScene.modified_description = currScene.scene_desc; + currScene.modified_ocr = ""; + currScene.has_ai = false; + currScene.deletedScene = []; + currScene.videoId = youtubeId; + console.log(sceneDataJson.scenes[key]); + } + + $.post("/saveNonAiDescription", sceneDataJson, function(response) { + if (response.success) { + alert( + "Submitted Successfully\nYou will be redirected to the Video List Page" + ); + window.history.back(); + } else { + alert("Update Failed: " + response.success); + } }); + }); }); -function convertToSeconds(time){ - var a = time.split(':'); - var seconds = (+a[0]) * 60 * 60 + (+a[1]) * 60 + (+a[2]); - return seconds; +function convertToSeconds(time) { + var a = time.split(":"); + var seconds = +a[0] * 60 * 60 + +a[1] * 60 + +a[2]; + return seconds; } function fetchDataFromAllFields() { - let listItems = $("#ul-scene-list li"); - let numItems = listItems.length; - let parentJsonObj = {}; - parentJsonObj.user_id = userId; - parentJsonObj.time_taken = timeElapsedOnPage; - let sceneArr = []; - for(let i=0; i + let sceneHtml = `
  • Scene Start Time
    - Scene ${$('#ul-scene-list li').length} + Scene ${$("#ul-scene-list li").length}

    (hh:mm:ss)

    - +
    • `; - $('#ul-scene-list').append(sceneHtml); - $('.sceneTimeEntry').timeEntry({ - spinnerImage: '../images/spinnerUpDown.png', - spinnerBigImage: '../images/spinnerUpDownBig.png', - spinnerSize: [15, 16, 0], - spinnerBigSize: [30, 32, 0], - unlimitedHours: true, - showSeconds: true, - spinnerIncDecOnly: true, - useMouseWheel: true - }); + $("#ul-scene-list").append(sceneHtml); + $(".sceneTimeEntry").timeEntry({ + spinnerImage: "../images/spinnerUpDown.png", + spinnerBigImage: "../images/spinnerUpDownBig.png", + spinnerSize: [15, 16, 0], + spinnerBigSize: [30, 32, 0], + unlimitedHours: true, + showSeconds: true, + spinnerIncDecOnly: true, + useMouseWheel: true + }); } function onDeleteButtonClicked(element) { - $(element).parent().parent().parent().remove(); - let listItems = $("#ul-scene-list li"); - for (let i=0; i currentTime) { - timeAtWhichLastPaused = currentTime; - } - if(sceneEndTime - currentTime < 0.1 && currentTime - timeAtWhichLastPaused > 0.5) { - pauseVideo(); - timeAtWhichLastPaused = currentTime; - //if(sceneToBlockMap.has(sceneId)) {//Check if it's the last scene of the current block - // let blockId = sceneToBlockMap.get(sceneId); - // let blockElementId = '#bl_cont_' + blockId; - } - let sceneElementId = '#sc_cont_' + sceneId; - $(previouslyHighlightedContainerId).removeClass('highlight'); - $(sceneElementId).addClass('highlight'); - previouslyHighlightedContainerId = sceneElementId; + currentTime = currentTime.toFixed(3); + let currScene = getSceneFromTime(currentTime, allSceneArr); + let sceneId = currScene.sc_id; + let sceneNum = currScene.sc_num; + let sceneEndTime = currScene.finish_time; + if (prevSceneNum != sceneNum) { + let listPosition = sceneIdToListPositionMap.get(sceneId); + scrollToPosition(listPosition); + } + prevSceneNum = sceneNum; + if (timeAtWhichLastPaused > currentTime) { + timeAtWhichLastPaused = currentTime; + } + if ( + sceneEndTime - currentTime < 0.1 && + currentTime - timeAtWhichLastPaused > 0.5 + ) { + pauseVideo(); + timeAtWhichLastPaused = currentTime; + //if(sceneToBlockMap.has(sceneId)) {//Check if it's the last scene of the current block + // let blockId = sceneToBlockMap.get(sceneId); + // let blockElementId = '#bl_cont_' + blockId; + } + let sceneElementId = "#sc_cont_" + sceneId; + $(previouslyHighlightedContainerId).removeClass("highlight"); + $(sceneElementId).addClass("highlight"); + previouslyHighlightedContainerId = sceneElementId; } function onPlayerStateChange(event) { - let playerState = event.data; - switch(playerState) { - case YT.PlayerState.PLAYING: - $("#btn-play-pause").html(''); - break; - case YT.PlayerState.PAUSED: - $("#btn-play-pause").html(''); - timeAtWhichLastPaused = player.getCurrentTime(); - // console.log(timeAtWhichLastPaused); - // console.log('i was called: paused state'); - break; - case YT.PlayerState.UNSTARTED: - case YT.PlayerState.ENDED: - case YT.PlayerState.CUED: - timeAtWhichLastPaused = player.getCurrentTime(); - $("#btn-play-pause").html(''); - // console.log('i was called: cued state') - break; - } + let playerState = event.data; + switch (playerState) { + case YT.PlayerState.PLAYING: + $("#btn-play-pause").html( + '' + ); + break; + case YT.PlayerState.PAUSED: + $("#btn-play-pause").html( + '' + ); + timeAtWhichLastPaused = player.getCurrentTime(); + // console.log(timeAtWhichLastPaused); + // console.log('i was called: paused state'); + break; + case YT.PlayerState.UNSTARTED: + case YT.PlayerState.ENDED: + case YT.PlayerState.CUED: + timeAtWhichLastPaused = player.getCurrentTime(); + $("#btn-play-pause").html( + '' + ); + // console.log('i was called: cued state') + break; + } } function playVideo() { - player.playVideo(); + player.playVideo(); } function pauseVideo() { - player.pauseVideo(); + player.pauseVideo(); } function stopVideo() { - player.stopVideo(); + player.stopVideo(); } async function updateAll(shoudShowAlert) { - let postJson = fetchJsonBodyForAllTextFields(allSceneArr, allBlockArr, userId); - await $.post('/updateMultipleScenesAndBlocksForUser', postJson, function (response) { - var currentdate = new Date(); - var datetime = "Last Sync: " + currentdate.getDate() + "/" - + (currentdate.getMonth()+1) + "/" - + currentdate.getFullYear() + " @ " - + currentdate.getHours() + ":" - + currentdate.getMinutes() + ":" - + currentdate.getSeconds(); - if(response.success) { - $('#txtSyncTime').html(datetime); - } - if(shoudShowAlert) { - if(response.success) { - alert('Submitted Successfully\nYou will be redirected to the Video List Page'); - window.history.back(); - } else { - alert('Update Failed: ' + response.success); - } - } - }); + let postJson = fetchJsonBodyForAllTextFields( + allSceneArr, + allBlockArr, + userId + ); + await $.post("/updateMultipleScenesAndBlocksForUser", postJson, function( + response + ) { + var currentdate = new Date(); + var datetime = + "Last Sync: " + + currentdate.getDate() + + "/" + + (currentdate.getMonth() + 1) + + "/" + + currentdate.getFullYear() + + " @ " + + currentdate.getHours() + + ":" + + currentdate.getMinutes() + + ":" + + currentdate.getSeconds(); + if (response.success) { + $("#txtSyncTime").html(datetime); + } + if (shoudShowAlert) { + if (response.success) { + alert( + "Submitted Successfully\nYou will be redirected to the Video List Page" + ); + window.history.back(); + } else { + alert("Update Failed: " + response.success); + } + } + }); } - async function updateMainTables() { - let postJson = fetchJsonBodyForAllTextFields2(allSceneArr, allBlockArr); - await $.post('/updateBlockAndScene', postJson, function (response) { - if(response.success) { - window.alert('Updated Successfully'); - } else { - window.alert('Update Failed'); - } - }); + let postJson = fetchJsonBodyForAllTextFields2(allSceneArr, allBlockArr); + await $.post("/updateBlockAndScene", postJson, function(response) { + if (response.success) { + window.alert("Updated Successfully"); + } else { + window.alert("Update Failed"); + } + }); } function fetchJsonBodyForAllTextFields(allSceneArr, allBlockArr, userId) { - let parentJsonBody = {}; - let sceneArrJson = []; - let blockArrJson = []; - for(let i=0; i { + // console.log("The vals are: "+sceneId+" "+videoTime); + res.send(result); + }); +}); + +router.get('/getKeyFrames', function(req, res, next) { + //let query = video.fetchKeyFrames(); + + let {videoId, videoTime} = req.query; + console.log("The vals are: "+videoId+" "+videoTime); + let queryResult = video.fetchKeyFrames(videoId, videoTime); + queryResult.then(result => { + console.log("The vals are: "+videoId+" "+videoTime); + res.send(result); + }); +}); + +router.get('/getRevisedKeyFrames', function(req, res, next) { + //let query = video.fetchKeyFrames(); + + let {videoId, videoTime} = req.query; + console.log("The vals are: "+videoId+" "+videoTime); + let queryResult = video.fetchRevisedKeyFrames(videoId,videoTime); + queryResult.then(result => { + res.send(result); + }); +}); + +router.get('/filterDescriptionBleu', function(req, res, next) { + let videoId = req.query.videoId; + if(!videoId){ + res.status(400).send('Missing parameter'); + }else{ + filterDescriptionBleuController.filter(videoId) + .then(result => { + res.status(200).send('SUCCESS: Data updated for the video in Scene table'); + }) + .catch(e => { + console.log(e); + res.send(e); + }) + } + +}); + + +router.get('/getDialogTimeStamps', function(req, res, next) { + //let query = video.fetchKeyFrames(); + console.log('Dialog TS:',req); + let {videoId} = req.query; + console.log("The vals are: "+videoId); + let queryResult = video.fetchDialogTimeStamps(videoId); + queryResult.then(result => { + console.log("The vals are: "+videoId); + res.send(result); + }); +}); + +router.get('/getSentences', function(req, res, next) { + //let query = video.fetchKeyFrames(); + console.log('Sentences ',req); + let {videoId, userId} = req.query; + console.log("The vals are: "+videoId+' and '+userId); + let queryResult = video.fetchSentences(videoId,userId); + queryResult.then(result => { + console.log("The vals are: "+videoId); + res.send(result); + }); +}); + + router.get('/playvideo/:youtubeId', function(req, res, next) { let isAiEnabled = req.query.ai === 'yes'; if(isAiEnabled) { res.render('withai', { title: 'Express' }); } else { + res.set('Access-Control-Allow-Origin', '*'); res.render('withoutai', { title: 'Express' }); } }); + +router.get('/videoSceneData', function(req, res, next) { + console.log('req.query', req.query) + let {videoid} = req.query; + video.fetchScene(videoid) + .then((data)=>{ + res.status(200).send(data); + }) + .catch(e=>{ + console.log(e); + res.status(400).send(e); + }) + +}); + router.get('/fetchVideoData', function(req, res, next) { console.log('req.query', req.query) let {videoid, userId} = req.query; @@ -69,6 +223,23 @@ router.get('/fetchVideoData', function(req, res, next) { }); }); +router.get('/videoData', function(req, res, next) { + console.log('req.query', req.query) + let {videoid, userId} = req.query; + let queryResult = video.fetchVideoData(videoid, userId); + let data= {'videoId': videoid, + 'userId': userId + }; + + queryResult.then(result => { + quesAns.getQuestionAnswerByUserId(data).then( qnaList =>{ + console.log({ resdata: qnaList}); + result.qnaList = qnaList; + res.send(result); + }); + }); +}); + router.post('/updateSceneDescription', function(req, res, next) { let inputJson = req.body; let queryResult = video.updateSceneDescription(inputJson); @@ -147,20 +318,21 @@ router.get('/userStudy2/users/', function (req, res) { function addModifiedScene (data, callback) { - // console.log('inside modified scene:', data); - let { has_ai, video_id, scene_id, user, block_id, scene_num, start_time, end_time, description, ocr, deletedScene=[]} = data; + console.log('inside modified scene:', data); + let { has_ai, video_id, scene_id, user, block_id, scene_num, start_time, end_time, description, ocr, deletedScene=[], note} = data; let descriptionId = user + '_' + video_id + '_ai_' + scene_num; - // console.log({INSIDEpSOT: data}); - // console.log({descriptionId}); + console.log({INSIDEpSOT: data}); + console.log({descriptionId}); let mergedSceneArr = JSON.stringify(deletedScene); description = description.replace(/'/g, ''); ocr = ocr.replace(/'/g, ''); + note = note.replace(/'/g, ''); console.log('MERGED SCENE ARR:', mergedSceneArr); const sql = ` START TRANSACTION; - INSERT INTO descriptions(desc_id, video_id, has_ai, volunteer_id, scene_id, scene_num, start_time, end_time, description, ocr, merged_scene) - VALUES('${descriptionId}', '${video_id}', '${has_ai}','${user}', '${scene_id}', ${scene_num}, ${start_time}, ${end_time}, '${description}', '${ocr}', '${mergedSceneArr}') - ON CONFLICT(scene_id, volunteer_id) DO UPDATE SET description ='${description}' , merged_scene = '${mergedSceneArr}', end_time= ${end_time}, start_time=${start_time}, ocr='${ocr}' ; + INSERT INTO descriptions(desc_id, video_id, has_ai, volunteer_id, scene_id, scene_num, start_time, end_time, description, ocr, merged_scene, note) + VALUES('${descriptionId}', '${video_id}', '${has_ai}','${user}', '${scene_id}', ${scene_num}, ${start_time}, ${end_time}, '${description}', '${ocr}', '${mergedSceneArr}', '${note}') + ON CONFLICT(scene_id, volunteer_id) DO UPDATE SET description ='${description}' , merged_scene = '${mergedSceneArr}', end_time= ${end_time}, start_time=${start_time}, ocr='${ocr}', note= '${note}'; COMMIT;`; // console.log({sql}); @@ -171,12 +343,23 @@ function addModifiedScene (data, callback) { async function deleteMergedScenes(sceneArr, videoId, user_id) { let sql = ''; for(let i=0; i< sceneArr.length; i++){ - sql += `DELETE from descriptions where scene_id = '${sceneArr[i]}' and video_id='${videoId}' and volunteer_id='${user_id}';`; + sql += `DELETE from descriptions where scene_id = '${sceneArr[i]}' and video_id='${videoId}' and volunteer_id='${user_id}';`; } console.log({'INSIDE DELETE': sql}) return db.query(sql, sceneArr); } +async function updateDeletedScenes(sceneArr, videoId, user_id) { + let sql = ''; + for(let i=0; i< sceneArr.length; i++){ + sql += `Update descriptions set isdeleted = true where scene_id = '${sceneArr[i]}' and video_id='${videoId}' and volunteer_id='${user_id}';`; + } + console.log({'INSIDE updateDeletedScenes': sql}) + return db.query(sql, sceneArr); +} + + + router.post('/updatefeedback', function(req, res, next) { let rating = req.body.rating; let comment = req.body.comment; @@ -236,10 +419,77 @@ router.post('/saveNonAiDescription', function(req, res, next) { }); +router.post('/saveModifiedDescription', function(req, res, next) { + let inputJson = req.body; + // console.log('inside saveModifiedDescription'); + console.log('inside saveModifiedDescription:', inputJson); + + let {qnaData, user_id, videoId} = inputJson; + //saving Q&A for user study phase 3 + let resArr = []; + for (let key in qnaData) { + if (qnaData.hasOwnProperty(key)) { + // console.log("KEY:", key); + for(let i= 0; i< qnaData[key].length; i++){ + // console.log("value" ," -> " , qnaData[key][i].question); + resArr = [...resArr, quesAns.addQuestionAnswer({...qnaData[key][i], user: user_id, sceneId: key})] + } + } + + } + + let deletedSceneArr = []; + for(let key in inputJson.scene_arr) { + + let currScene = inputJson.scene_arr[key]; + console.log({currScene}); + let {scene_id, modified_description, modified_ocr, start_time, end_time, has_ai, deletedScene} = currScene; + if(deletedScene){ + deletedSceneArr = [...deletedSceneArr, ...deletedScene]; + } + + console.log({deletedSceneArr}); + resArr = [...resArr, addModifiedScene({ + scene_id, + user: user_id, + scene_num: key, + video_id: req.body.videoId, + description: modified_description, + ocr: modified_ocr, + start_time, + end_time, + has_ai, + deletedScene + })]; + } + + console.log({deletedSceneArr}); + + Promise.all(resArr) + .then(() => { + if(deletedSceneArr.length > 0){ + updateDeletedScenes(deletedSceneArr, videoId, user_id) + .then((response)=>{ + let queryResult = db.query(`INSERT INTO temp_table_ai(temp_column) VALUES($1)`,[inputJson]); + console.log({inputJson}) + queryResult.then(result => { + res.send({success: true}); + }) + }) + }else{ + res.status(200).send('Data saved succefully'); + } + }) + .catch((e) => { + console.log(e); + res.status(400).send(e); + }) +}); router.post('/saveAiDescription', function(req, res, next) { let inputJson = req.body; + console.log({inputJson}); console.log('SAVE AI DESCRiption'); // console.log('TEST Save AI:', inputJson); let {qnaData, user_id, videoId} = inputJson; @@ -260,8 +510,11 @@ router.post('/saveAiDescription', function(req, res, next) { for(let key in inputJson.scene_arr) { let currScene = inputJson.scene_arr[key]; - console.log({currScene}); - let {scene_id, modified_description, modified_ocr, start_time, end_time, has_ai, deletedScene} = currScene; + // console.log({currScene}); + let {scene_id, modified_description, modified_ocr, start_time, end_time, has_ai, deletedScene, note} = currScene; + if(!note){ + note = ''; + } if(deletedScene){ deletedSceneArr = [...deletedSceneArr, ...deletedScene]; } @@ -277,11 +530,12 @@ router.post('/saveAiDescription', function(req, res, next) { start_time, end_time, has_ai, - deletedScene + deletedScene, + note })]; } - console.log({deletedSceneArr}); + // console.log({deletedSceneArr}); Promise.all(resArr) @@ -308,7 +562,14 @@ router.post('/saveAiDescription', function(req, res, next) { router.post('/texttospeech', function(req, res, next) { let inputJson = req.body; let inputText = inputJson.inputText; - let task = textToSpeech.generateMp3ForText(inputText); + let filePath = inputJson.filePath; + let task = ''; + if(filePath){ + filePath = "public/audio/" + filePath; + task = textToSpeech.generateMp3ForTextInFileystem(filePath,inputText); + } + else + task = textToSpeech.generateMp3ForText(inputText); task.then(result => { res.send({success: result}); }); @@ -321,6 +582,90 @@ router.get('/generateMp3FromDescriptions', function(req, res, next) { res.send({success: result}); }); }); + // texttospeech -> duration +router.post('/descriptionDuration', function(req, res, next) { + let inputJson = req.body; + let inputText = inputJson.inputText; + let task = textToSpeech.generateMp3DurationForText(inputText); + task.then(result => { + // res.send(result.file); + res.send({success: result.success, duration: result.duration, file:result.file, form:result.form}); + }); +}); + +router.post('/descriptionMp3Audio', function(req, res, next) { + let inputJson = req.body; + let inputText = inputJson.inputText; + let task = textToSpeech.getMp3ForDescription(inputText, res); + task.then(result => { console.log(result) }); +}); + + /* For sentence generation starts*/ +router.get('/generateSentencesFromDescriptions', function(req, res, next) { + let task = sentences.generateSentencesForAllDescriptionsInDb(); + task.then(result => { + res.send({success: result}); + }); +}); + +router.get('/generateMp3ForAllSentences', function(req, res, next) { + let task = sentences.generateMp3ForAllSentencesInDb(); + task.then(result => { + res.send({success: result}); + }); +}); + +router.get('/generateTimestampsforSentences', function(req, res, next) { + let task = sentences.generateTimestampsforSentencesInDb(); + task.then(result => { + res.send({success: result}); + }); +}); + +router.get('/generateTimestampsforSentencesInDbWithInfo', function(req, res, next) { + let inputJson = req.query; + console.log('input: ',inputJson); + let {vid_id,vol_id} = inputJson; + console.log('GET params: ',vid_id,' and: ',vol_id); + let task = sentences.generateTimestampsforSentencesInDbWithInfo(vid_id,vol_id); + task.then(result => { + res.send({success: result}); + }); +}); + +router.get('/mergeSentencesInDb', function(req, res, next) { + let inputJson = req.query; + console.log('input: ',inputJson); + let {vid_id, vol_id} = inputJson; + console.log('GET params: ',vid_id); + let task = sentences.mergeSentenceAudios(vid_id, vol_id); + task.then(result => { + res.send({success: result}); + }); +}); + +router.get('/generateOCRTimestampsInDb', function(req, res, next) { + let inputJson = req.query; + console.log('input: ',inputJson); + let {vid_id, vol_id} = inputJson; + let task = audioDescriptionOCR.generateAudioDescriptionsForOCRInDbWithInfo(vid_id, vol_id, res); + task.then(result => { + res.send({success: result}); + }); +}); + +router.get('/mergeOCRExtendedAudio', function(req, res, next) { + let inputJson = req.query; + console.log('input: ',inputJson); + let {vid_id, vol_id} = inputJson; + let task = audioDescriptionOCR.mergeOCRExtendedAudio(vid_id, vol_id); + task.then(result => { + res.send({success: result}); + }); +}); + + + /* For sentence generation ends */ router.get('/playVideoAudio/:youtubeId', function(req, res, next) { res.render('videoaudio'); @@ -344,4 +689,150 @@ router.get('/getDescriptionData', function(req, res, next) { }); }); +// router.get('/getBleuScore', function(req, res, next){ +// let reference = req.query.reference; +// let sentence = req.query.sentence; +// console.log('ref: ',reference, ' and: ',sentence); +// let task = bleu.generateBLEUScore(reference,sentence); +// task.then(result => { +// res.send({'result': result}); +// }); +// }); + +router.get('/refreshAudioData', function(req, res, next){ + let videoId= req.query.videoId; + let volunteerId = req.query.volunteerId; + + console.log('vid: '+videoId+' vol: '+volunteerId); +let task = sentences.refereshAudioData(videoId,volunteerId); +task.then(result => { +res.send({'result': result}); +}); +}); + + +router.get('/generateAudioDescriptionMp3', function(req, res, next){ + let videoId= req.query.videoId; + let volunteerId = req.query.volunteerId; + let merge = false; + let hasOcr = false; + if(req.query.merge != null) + merge = true; + if(req.query.hasOcr != null) + hasOcr = true; + + console.log('vid: '+videoId+' vol: '+volunteerId); +let task = audioDescription.generateAudioDescriptionsInDbWithInfo(videoId,volunteerId,merge,hasOcr, res); +task.then(result => { +res.send({'result': result}); +}); +}); + +router.get('/getAudioDescriptionMp3', function(req, res, next){ + let videoId= req.query.videoId; + let volunteerId = req.query.volunteerId; + console.log('vid: '+videoId+' vol: '+volunteerId); +let task = audioDescription.getResponse(videoId,volunteerId, res); +task.then(result => { +res.send({'result': result}); +}); +}); + +router.post('/saveAudioDescriptionTimelineForSentence', function(req, res, next){ + let videoId= req.body.videoId; + let volunteerId = req.body.volunteerId; + let sentenceId = req.body.sentenceId; + let audioType = req.body.audioType; + let audioStartTime = req.body.audioStartTime; + + console.log('vid: '+videoId+' vol: '+volunteerId); +let task = audioDescription.saveAudioDescriptionTimelineForSentence(videoId,volunteerId, sentenceId, audioType, audioStartTime, res); +}); + +router.post('/saveAudioDescriptionTimelineForScene', function(req, res, next){ + let videoId= req.body.videoId; + let volunteerId = req.body.volunteerId; + let sentences = req.body.sentences; + let audioType = req.body.audioType; + let audioStartTime = req.body.audioStartTime; + + console.log('vid: '+videoId+' vol: '+volunteerId); +let task = audioDescription.saveAudioDescriptionTimelineForScene(videoId,volunteerId, sentences, audioType, audioStartTime, res); +}); + +router.get('/generateAudioDescriptionMp3ForScene', function(req, res, next){ + let videoId= req.query.videoId; + let volunteerId = req.query.volunteerId; + let sceneId = req.query.sceneId; + let merge = false; + if(req.query.merge != null) + merge = true; + + console.log('vid: '+videoId+' vol: '+volunteerId); +let task = audioDesc.generateAudioDescriptionsInDbWithInfo(videoId,volunteerId,sceneId, res); +task.then(result => { +res.send({'result': result}); +}); +}); + +router.get('/mergeAudioDescriptionMp3', function(req, res, next){ + let videoId= req.query.videoId; + let volunteerId = req.query.volunteerId; + + console.log('vid: '+videoId+' vol: '+volunteerId); +let task = audioDescription.mergeSentenceAudios(videoId,volunteerId, res); +task.then(result => { +res.send({'result': result}); +}); +}); + +// generateAudioDescriptionsInDbWithInfo +router.post('/audioDescriptionByScene', function(req, res, next) { + let inputJson = req.body; + let video_id = inputJson.vid_id; + let volunteer_id = inputJson.vol_id; + let scene_id = inputJson.scene_id; + let description = inputJson.desc; + let task = audioDescription.generateTimestampsforScene(video_id, volunteer_id, scene_id, description); + task.then(result => { + res.send({"status":result}); + }); +}); + +router.get('/generateTestAudio', function(req, res, next){ + let videoId= req.query.videoId; + let volunteerId = req.query.volunteerId; + + console.log('vid: '+videoId+' vol: '+volunteerId); +let task = audioDescription.generateTimestampsforSentencesInDbWithInfo(videoId,volunteerId); +task.then(result => { +res.send({'result': result}); +}); +}); + +// router.use(bodyParser.urlencoded({ extended: false })); +// router.use("/", expressStaticGzip("dist")); +// router.onBeforeAction(Iron.router.bodyParser.urlencoded({ +// extended: false +// })); + + + + + + +router.post('/splitMp3', function(req, res, next) { + let inputJson = req.body; + console.log(req.body); + console.log(req.headers['content-type']); + let file = inputJson.file; + let time = inputJson.time; + let callbackURL = inputJson.callbackURL; + let task = sentences.mp3Split(file, time, callbackURL); + task.then(result => { + res.send({"status":result}); + }); +}); + + module.exports = router; diff --git a/routes/keyframeUpdate.js b/routes/keyframeUpdate.js new file mode 100644 index 0000000..7b4b42f --- /dev/null +++ b/routes/keyframeUpdate.js @@ -0,0 +1,47 @@ +/** + * description: consists of the routes related to revised/new keyframes + * Basic route: localhost:5001/keyframe + */ + +var express = require('express'); +var router = express.Router(); +const userStudy = require('../model/userStudy'); +const userStudy4 = require('../model/userStudy4'); +const questionUserStudy4 = require('../model/questionUserStudy4'); +const db = require('../db/database'); +const uuidv1 = require('uuid/v1'); +const fs = require('fs'); +var path = require('path'); +const youtubedl = require('youtube-dl'); +let location1= 'Trial'; +const axios = require('axios'); +const FormData = require('form-data'); +const fetch = require('node-fetch'); +var request = require('request'); +const keyframeRevise = require('../model/keyframeRevise'); +let accountId = '5a8c6ddf-900f-48b4-8a3a-0cd5cf48cb4b'; +let apiKey = 'dbbd2d92a19749759576466ed9cd61fa'; + + +router.post('/addKeyframe', function (req, res) { + console.log("Add keyframe body: ",req.body); + let queryResult = keyframeRevise.addKeyframe(req.body); + queryResult.then(result => { + res.send('Keyframe inserted successfully'); + }).catch((err) => { + res.send(err); + console.log(err) + }); + }); + +router.post('/updateKeyframe', function(req, res, next) { + let queryResult = keyframeRevise.updateKeyframe(req.body); + queryResult.then(result => { + res.send('Keyframe updated successfully'); + }).catch((err) => { + res.send(err); + console.log(err) + }); + }); + + module.exports = router; \ No newline at end of file diff --git a/routes/mergeScene.js b/routes/mergeScene.js new file mode 100644 index 0000000..6959f4e --- /dev/null +++ b/routes/mergeScene.js @@ -0,0 +1,53 @@ +var express = require('express'); +var router = express.Router(); +const scene = require('../model/scene'); +const fs = require('fs'); +let UploadVideoController = require('../controllers/UploadVideoController'); + +router.get('/mergeAudioVideo', function (req, res) { + let videoId = req.query.videoId; + let audioId = req.query.audioId; + let queryResult = UploadVideoController.fetchAudioInsight(audioId, videoId); + queryResult + .then(data => { + res.send(data); + // console.log({data}); + }) + .catch((err) => { + res.status(400).send(err); + }); +}); + + +router.get('/sceneData', function (req, res) { + let videoId = req.query.videoId; + let queryResult = scene.getSceneData(videoId); + queryResult + .then(data => { + res.send(data); + // console.log({data}); + }) + .catch((err) => { + res.send(err); + }); + }); + + router.post('/scene', function (req, res) { + let sceneArr = req.body.mergedList; + + console.log({sceneArr}); + let queryResult = scene.addMergeSceneTable(sceneArr); + queryResult + .then(data => { + console.log({data}); + res.status(200).send('Data added successfully in merge_scene table'); + + }) + .catch((err) => { + console.log('Post scene Data Error: ', err); + res.status(400).send(err); + }); + }); + + +module.exports = router; \ No newline at end of file diff --git a/routes/pythiaCaption.js b/routes/pythiaCaption.js new file mode 100644 index 0000000..d6f1c86 --- /dev/null +++ b/routes/pythiaCaption.js @@ -0,0 +1,25 @@ +const express = require('express'); +const router = express.Router(); +const axios = require('axios'); +const pythiaCaptionController = require('../controllers/PythiaCaptionGenerationController'); + + + +router.get('/generateCaption', function (req, res) { + let videoId = req.query.videoId; + if (videoId) { + let queryResult = pythiaCaptionController.generatePythiaCaption(videoId); + queryResult + .then(result => { + res.status(200).send('Pythia captions successfully updated'); + console.log(result); + }) + .catch((err) => { + res.status(400).send(err); + }); + }else{ + res.status(400).send('Missing request parameter'); + } +}); + +module.exports = router; \ No newline at end of file diff --git a/routes/question_answer.js b/routes/question_answer.js index 2f5edb4..9f8a012 100644 --- a/routes/question_answer.js +++ b/routes/question_answer.js @@ -49,9 +49,9 @@ router.post('/addQuestion', function(req, res, next) { router.delete('/deleteQuestion', function(req, res, next) { let queryResult = questionAnswer.deleteQuestionById(req.body); queryResult.then(result => { - res.send('Question deleted successfully'); + res.status(200).send('Question deleted successfully'); }).catch((err) => { - res.send(err); + res.status(400).send(err); console.log(err) }); }); @@ -64,7 +64,7 @@ router.post('/generate', async function(req, res) { let result = []; let gotResponse = 0; for(let i=0; i < rows.length; i++){ - axios.post('http://localhost:5000/generate', {sentence: rows[i]}).then((response) => { + axios.post('http://localhost:5300/generate', {sentence: rows[i]}).then((response) => { // console.log("success", response.data) gotResponse++; result.push({response: response.data}) diff --git a/routes/uploadVideo.js b/routes/uploadVideo.js new file mode 100644 index 0000000..32627c7 --- /dev/null +++ b/routes/uploadVideo.js @@ -0,0 +1,509 @@ +/** + * author: vaishali + * description: consists of all routes related to integration of Video Indexer + * Basic route: localhost:5001/userStudy + */ + +var express = require('express'); +var router = express.Router(); +const userStudy = require('../model/userStudy'); +const db = require('../db/database'); +const fs = require('fs'); +var path = require('path'); +const youtubedl = require('youtube-dl'); +const axios = require('axios'); +const FormData = require('form-data'); +const videoController = require('../backend/video'); +let accountId = '5a8c6ddf-900f-48b4-8a3a-0cd5cf48cb4b'; +let apiKey = 'dbbd2d92a19749759576466ed9cd61fa'; +let location = 'trial'; +var ffmpeg = require('ffmpeg'); +let scene = require('../model/scene'); +let videoModel = require('../model/video.js'); +const request = require('request'); +let videoDialogue = require('../model/videoDialogue'); +let uploadVideoController = require('../controllers/VideoControllerUpload'); + + +async function downloadVideo(url, videoTitle){ + let videoId = url.substr(url.indexOf('v=') + 2); + console.log({ videoTitle}); + + + return new Promise((resolve, reject) => { + let jsonPath = path.join(__dirname, '../videos/'); + console.log({ jsonPath }); + console.log({ __dirname }); + const video = youtubedl(url, ['--format=18']); + // const video = youtubedl(url, ['--format=137/22/18']); + + // Will be called when the download starts. + video.on('info', function (info) { + console.log('Download started') + console.log('filename: ' + info._filename) + console.log('size: ' + info.size) + }) + + let date_ob = new Date(); + let year = date_ob.getFullYear(); + let month = ("0" + (date_ob.getMonth() + 1)).slice(-2); + let date = ("0" + date_ob.getDate()).slice(-2); + let currentDateTime = year + "-" + month + "-" + date + "_" + Math.floor(Date.now() / 1000); + + let rootName = videoId + '_' + currentDateTime; + let fileName = videoId + '_' + currentDateTime + '.mp4'; + let filePath = path.join(__dirname, '../videos/', fileName); + console.log({ fileName }); + console.log({ filePath }); + + video.pipe(fs.createWriteStream(filePath)); + video.on('complete', function complete(info) { + 'use strict' + console.log('filename: ' + info._filename + ' already downloaded.') + }) + let name = ''; + video.on('end', function () { + console.log('Finished downloading!'); + resolve('Video Downloaded successfully'); + }) + + video.on('error', function error(err) { + console.log('error 2:', err); + reject(err); + }) + + }) + + } + + + + + +router.get('/downloadVideo', function(req, res) { + let {videoUrl, title} = req.query; + console.log(req.query); + + downloadVideo(videoUrl, title) + .then(()=>{ + res.status(200).send('Video downloaded successfully'); + }) + .catch((e)=>{ + res.status(400).send('Failed to download the video'); + console.log(e); + }) +}); + +router.get('/audioInsight', function(req, res, next) { + console.log('req.query', req.query); + let audioId = req.query.order; + let status = req.query.status; + if(status === '-1'){ + res.status(400).send('Error: ListenByCode failed to process the audio file'); + }else{ + let url = 'http://18.221.192.73:8081/v1/audioclips/getspeechtotextresult?orderid=' + audioId; + axios.get(url) + .then((response) => { + console.log(response.data); + //AUDIO + let sentence = response.data.result.sentence; + let audioInsight =[]; + for(let i=0; i< sentence.length; i++){ + let start = sentence[i].startTime; + let end = sentence[i].endTime; + let data = { + start, + end + } + audioInsight.push(data); + } + console.log({audioInsight}); + + //VIDEO + let videoInsight = []; + videoModel.getSceneByAudioId(audioId) + .then((data) => { + console.log(data); + for (let i = 0; i < data.length; i++) { + data[i].merged = []; + videoInsight.push(data[i]); + } + console.log({ videoInsight }); + + //MERGE + let mergedList = uploadVideoController.merge(videoInsight, audioInsight); + let result = scene.addMergeSceneTable(mergedList) + result + .then(data => { + console.log({ success: data }); + res.status(200).send('Data merged successfully'); + }) + .catch((err) => { + console.log(err); + res.status(400).send(err); + }); + + }).catch((error) => { + console.log(error); + res.status(400).send(error); + }) + + }) + .catch((error) => { + console.log(error); + res.status(400).send(error); + }) + } + +}); + + +router.get('/videoState', function (req, res) { + console.log('INSIDe GET video State'); + let videoId = req.query.id; + let state = req.query.state; + console.log(req.query); + + if (state === 'Processed') { + videoModel.getVideoAudioByVideoId(videoId) + .then((data)=>{ + console.log({data}); + let youtubeId = data[0].youtube_id; + audioId = data[0].audio_id; + console.log({youtubeId, audioId }); + + let url = `https://api.videoindexer.ai/${location}/accounts/${accountId}/videos/${videoId}/Index/?language=en-US`; + console.log({ url }); + + axios.get(url) + .then((response) => { + // console.log(response.data); + let jsonBody = { + youtubeId, + 'inputJson': response.data + }; + //--- + videoController.addVideoDataToDb(jsonBody) + .then(result => { + console.log({ success: result }); + uploadVideoController.fetchAudioInsight(audioId, youtubeId) + .then((audioResult)=>{ + uploadVideoController.fetchAndFilterPythiaCaption(youtubeId) + .then(()=>{ + console.log(result); + let data = { + state: 'Processed', + youtubeId, + } + videoModel.updateVideoAudio_videoState(data) + .then(()=>{ + console.log('Video is processed and state updated in video_audio table'); + res.status(200).send('Video successfully processed'); + }) + }) + }) + }) + .catch((e)=>{ + console.log(e); + }) + //------ + }) + .catch((error) => { + res.status(400).send(error); + console.log(error); + }) + }).catch((err)=>{ + res.send(err); + }) + }else { + res.status(400).send('Video is not processed by Video Indexer , id:', videoId); + console.log('Video is not processed by Video Indexer , id:', videoId); + } +}); + +router.get('/videoIndexedState', function(req,res){ + let youtubeId = req.query.videoId; + videoModel.fetchVideoAudioTable({youtubeId}) + .then((result)=>{ + if(result.length == 0){ + res.status(404).send('Video not found'); + }else{ + let videoState = result[0].video_state; + res.send({videoState}); + console.log({result}); + } + + + }).catch(error => { + res.status(400).send(error); + console.log(error); + }) +}); + + +router.get('/uploadVideo', function (req, res) { + console.log({ param: req.query }); + let url = req.query.video; + let videoTitle = req.query.title; + + if(!url || !videoTitle){ + res.status(400).send('Missing request parameter'); + } + let videoId = url.substr(url.indexOf('v=') + 2); + + // let jsonPath = path.join(__dirname, '../videos/'); + // console.log({ jsonPath }); + // console.log({ __dirname }); + const video = youtubedl(url, ['--format=18']); + + // Will be called when the download starts. + video.on('info', function (info) { + console.log('Download started') + console.log('filename: ' + info._filename) + console.log('size: ' + info.size) + }) + + let date_ob = new Date(); + let year = date_ob.getFullYear(); + let month = ("0" + (date_ob.getMonth() + 1)).slice(-2); + let date = ("0" + date_ob.getDate()).slice(-2); + let currentDateTime = year + "-" + month + "-" + date + "_" + Math.floor(Date.now() / 1000); + + let rootName = videoId + '_' + currentDateTime; + let fileName = videoId + '_' + currentDateTime + '.mp4'; + let filePath = path.join(__dirname, '../videos/', fileName); + console.log({ fileName }); + console.log({ filePath }); + + video.pipe(fs.createWriteStream(filePath)); + video.on('complete', function complete(info) { + 'use strict' + console.log('filename: ' + info._filename + ' already downloaded.') + }) + let name = ''; + video.on('end', function () { + console.log('Finished downloading!'); + + let tokenUrl = `https://api.videoindexer.ai/Auth/trial/Accounts?generateAccessTokens=true&allowEdit=true`; + axios({ + method: 'get', + url: tokenUrl, + headers: { 'Ocp-Apim-Subscription-Key': apiKey }, + }) + .then(response => { + let accessToken = response.data[0].accessToken; + console.log('token->' + accessToken); + uploadVideo(filePath, accessToken, videoId, videoTitle) + .then((videoData) => { + console.log({ videoData }); + let data = { + youtubeId: videoId, + video_id: videoData.id, + title: videoTitle, + videoState: 'Processing' + } + console.log({ videoAudioData: data }); + + videoModel.insertIntoVideoAudioTable(data) + .then(() => { + convertVideoToAudio(videoId, filePath) + .then((audioData) => { + console.log({ audioData }); + let audioId = audioData.audioId; + //---------------- + let timeId = setInterval( ()=> { + let url = `https://api.videoindexer.ai/${location}/accounts/${accountId}/videos/${videoData.id}/Index/?language=en-US`; + console.log({ url }); + + axios.get(url) + .then((response) => { + let videoState = response.data.state; + console.log('Video State: ', videoState); + if(videoState === 'Processed'){ + let jsonBody = { + youtubeId: videoId, + 'inputJson': response.data + }; + clearInterval(timeId); + videoController.addVideoDataToDb(jsonBody) + .then(result => { + console.log({ success: result }); + uploadVideoController.fetchAudioInsight(audioId, videoId) + .then((audioResult)=>{ + uploadVideoController.fetchAndFilterPythiaCaption(videoId) + .then(()=>{ + console.log(result); + let data = { + state: 'Processed', + youtubeId: videoId, + } + videoModel.updateVideoAudio_videoState(data) + .then(()=>{ + console.log('Video is processed and state updated in video_audio table') + }) + }) + }) + }) + .catch((e)=>{ + console.log(e); + }) + } + }) + .catch((error) => { + clearInterval(timeId); + let data = { + state: 'Failed', + youtubeId: videoId, + } + videoModel.updateVideoAudio_videoState(data) + .then(()=>{ + console.log('video state updated in video_audio table') + }) + console.log(error); + }) + }, 10000); + //------------------------------------ + + res.status(200).send('SUCCESS: Video uploaded to Video Indexer'); + console.log('SUCCESS: Video uploaded to Video Indexer'); + }) + }) + }) + .catch(uploadVideoError => { + let data = { + state: 'Failed', + youtubeId: videoId, + } + videoModel.updateVideoAudio_videoState(data) + .then(()=>{ + console.log('video state updated in video_audio table') + }) + + console.log(uploadVideoError) ; + }) + }) + .catch(error => { + res.status(400).send('FAILED: Video not uploaded to Video Indexer '); + console.log(error); + }); + }) + + video.on('error', function error(err) { + console.log('error 2:', err); + res.status(400).send(err); + }) + +}); + + +async function convertVideoToAudio(youtubeId, videoFilePath) { + let index1 = videoFilePath.lastIndexOf('.mp4'); + let index2 = videoFilePath.lastIndexOf('/'); + let videoId = videoFilePath.substring(index1 + 1, index2); + let rootPath = videoFilePath.substring(0, index2); + + let audioFile = rootPath + videoId + 'mp3'; + console.log({ rootPath }); + console.log({ audioFile }); + console.log({ videoFilePath }); + + + return new Promise((resolve, reject) => { + let process = new ffmpeg(videoFilePath); + process.then((video) => { + console.log('VIDEO:' ,video.metadata); + video.fnExtractSoundToMP3(audioFile, + function (error, file) { + if (error) { + console.log('Error converting video to audio file'); + reject(error); + } else { + console.log('video extraction succeeded:', file); + ///---- + // let audioFile = '/Users/vaishalibisht/Desktop/SFSU/YouDescribe/usr-yd-prototype/GXWpEjEFWkM_2020-04-02_1585852150.mp3'; + let readStream = fs.createReadStream(audioFile); + readStream.on('error', function (err) { + console.log(err); + reject(err); + }); + + const formData = { + appkey: 'yAOzHAy9LQBJQGtshcIGJX368IbC4Enx', + language: "en", + file: readStream, + callbackurl: '', + }; + + request.post({url:"https://www.listenbycode.com/api/v1/upload", formData}, function optionalCallback(err, response, body) { + var jsonObj = JSON.parse(body); + var audioId = jsonObj.result.orderId; + console.log(body); + let data = { + audioId, + youtubeId + } + console.log(data); + videoModel.updateAudioId_VideoAudio(data) + .then((result)=>{ + console.log(result); + resolve(data); + }) + .catch((e)=>{ + console.log('Error while uploading audio file to SpeechTotextApi:', err); + reject(err); + }) + }); + + } + }) + }) + .catch((err) => { + console.log('Convert mp4 to mp3 error:' + err); + reject(err); + }); + }); +} + + + +async function uploadVideo(file, accessToken, youtubeId, videoTitle) { + console.log('inside upload video'); + console.log({ file }); + let name = youtubeId; + // let callbackUrl = `http://18.221.192.73:5001/vi/videoState?youtubeId=${youtubeId}`; + let callbackUrl = ''; + let url = `https://api.videoindexer.ai/trial/Accounts/${accountId}/Videos?name=${videoTitle}&privacy=Public&callbackUrl=${callbackUrl}&indexingPreset=Default&streamingPreset=Default&sendSuccessEmail=False&accessToken=${accessToken}`; + let formdata = new FormData(); + + let rs = fs.createReadStream(file); + rs.on('error', function (err) { + console.log(err) + }); + + formdata.append('file', fs.createReadStream(file)); + console.log({ callbackUrl }); + const config = { + method: 'post', + url, + data: formdata, + maxContentLength: Infinity, + maxBodyLength: Infinity, + headers: formdata.getHeaders() + } + + try{ + res = await axios(config); + console.log(res.data); + }catch(e){ + console.log('Video Indexer error: ', e); + throw e; + } + + return res.data; +} + + + +module.exports = router; + + diff --git a/routes/userStudy.js b/routes/userStudy.js index 73886f5..4506b62 100644 --- a/routes/userStudy.js +++ b/routes/userStudy.js @@ -8,9 +8,19 @@ var express = require('express'); var router = express.Router(); const userStudy = require('../model/userStudy'); const db = require('../db/database'); +const fs = require('fs'); +var path = require('path'); +const youtubedl = require('youtube-dl'); +let location1= 'Trial'; +const axios = require('axios'); +const FormData = require('form-data'); +const fetch = require('node-fetch'); +var request = require('request'); +let accountId = '5a8c6ddf-900f-48b4-8a3a-0cd5cf48cb4b'; +let apiKey = 'dbbd2d92a19749759576466ed9cd61fa'; -router.get('/', function (req, res) { +router.get('/key=ceb60ef5-cb4f-432c-bb19-f0e26530a3ca', function (req, res) { res.render('homeUserStudy', { title: 'User Study' }); }); @@ -145,4 +155,8 @@ router.delete('/deleteScene', function(req, res, next) { console.log(err) }); }); -module.exports = router; \ No newline at end of file + + +module.exports = router; + + diff --git a/routes/userStudy4.js b/routes/userStudy4.js new file mode 100644 index 0000000..01aff58 --- /dev/null +++ b/routes/userStudy4.js @@ -0,0 +1,295 @@ +/** + * description: consists of all routes related to userstudy4 (summer 2020, on-demand) + * Basic route: localhost:5001/userStudy4 + */ + +var express = require('express'); +var router = express.Router(); +const userStudy = require('../model/userStudy'); +const userStudy4 = require('../model/userStudy4'); +const questionUserStudy4 = require('../model/questionUserStudy4'); +const db = require('../db/database'); +const uuidv1 = require('uuid/v1'); +const fs = require('fs'); +var path = require('path'); +const youtubedl = require('youtube-dl'); +let location1= 'Trial'; +const axios = require('axios'); +const FormData = require('form-data'); +const fetch = require('node-fetch'); +var request = require('request'); +let accountId = '5a8c6ddf-900f-48b4-8a3a-0cd5cf48cb4b'; +let apiKey = 'dbbd2d92a19749759576466ed9cd61fa'; + + +router.get('/key=ceb60ef5-cb4f-432c-bb19-f0e26530a3ca', function (req, res) { + res.render('userStudy4/homeUserStudy4', { title: 'User Study (On-Demand)' }); + }); + + //list of video for a user +router.get('/video', function (req, res) { + res.render('userStudy4/videoUserStudy4', { title: 'User Study Video' }); +}); + +router.get('/tutorialVideo', function (req, res){ + let userId = 'on-demand-tutorial' + let queryResult = db.query(`SELECT u.user_id, u.video_id, v.title, u.volunteer_id, u.ai_support, u.on_demand_support, usr.name FROM us_user_video u JOIN video v ON u.video_id = v.youtube_id JOIN us_user usr ON usr.uuid = u.user_id WHERE u.user_id = $1 ORDER by u.id`,[userId]); + queryResult + .then(result => { + res.send(result); + }) + .catch((err) => { + res.send(err); + }); + +}); + +//list of video for visually impaired users +router.get('/videoHelper', function (req, res) { + res.render('videoHelper', { title: 'User Study Video Helper' }); +}); + + +//list of visually impaired users +router.get('/helper/video', function (req, res) { + // console.log({ param: req.query.userId }); + let userId = req.query.userId; + if (userId) { + let queryResult = userStudy.getHelperVideo(userId); + queryResult + .then(result => { + res.send(result); + }) + .catch((err) => { + res.send(err); + }); + } +}); + + +router.get('/helperStatus', function (req, res) { + let userId= req.query.userId; + console.log({ userId}); + + if (userId) { + let queryResult = db.query(`SELECT * from us_helper_video where vi_user_id = '${userId}' and rating IS NOT NULL`); + queryResult + .then(result => { + res.json(result); + }) + .catch((err) => { + res.send({err}); + }); + } + }); + + router.get('/getUserHelper', function (req, res) { + let userId = req.query.userId; + let queryResult = userStudy.getAllUser_VI(userId); + queryResult + .then(result => { + res.send(result); + }) + .catch((err) => { + res.send(err); + }); + }); + + router.put('/updateStatus', function (req, res, next) { + let task = userStudy.updateStatus_VIUser(req.body); + task.then(result => { + res.status(200).json({ 'message': 'Thank you for you participation' }); + }) + .catch((err) => { + console.log(err) + res.send({ success: false }); + });; + }); + + router.put('/updateSceneEndTime', function (req, res, next) { + let task = userStudy.updateSceneEndTime(req.body); + task.then(result => { + res.status(200).json({succes : 'Scene end time updated successfully'}); + }) + .catch((err) => { + console.log(err) + res.send({ success: false }); + });; + }); + + router.put('/updateUserInfo', function (req, res, next) { + let task = userStudy.updateVIUserInfo(req.body); + task.then(result => { + res.status(200).json({ 'success': 'true' }); + }) + .catch((err) => { + console.log(err) + res.send({ success: false }); + });; + }); + + + + //list of users +//list of users study 1 +router.get('/users/', function (req, res) { + console.log({param: req.query.userId}); + let userId = req.query.userId; + if(userId){ + //let queryResult = userStudy.getVideoByUserId(userId); + let queryResult = db.query(`SELECT u.user_id, u.video_id, v.title, u.volunteer_id, u.ai_support, u.on_demand_support, usr.name FROM us_user_video u JOIN video v ON u.video_id = v.youtube_id JOIN us_user usr ON usr.uuid = u.user_id WHERE u.user_id = $1 ORDER by u.id`,[userId]); + queryResult + .then(result => { + res.send(result); + }) + .catch((err) => { + res.send(err); + }); + }else{ + let queryResult = userStudy4.getAllUsers(); + console.log({ queryResult }); + queryResult + .then(result => { + res.send(result); + }) + .catch((err) => { + res.send(err); + }); + } + }); + + router.get('/checkTutorial',function(req, res, next) { + console.log({param: req.query.userId}); + let userId = req.query.userId; + if(userId){ + + let queryResult = db.query(`SELECT u.is_tutorial_complete FROM us_user u WHERE u.uuid = $1`,[userId]); + queryResult + .then(result => { + console.log({result}); + res.send(result); + }) + .catch((err) => { + console.log({err}); + res.send(err); + }); + } + else{ + res.send({status : "error"}); + } + }); + + /** Updates tutorial as complete for a user */ + router.post('/updateTutorial',function(req, res, next) { + console.log({param: req.query.userId}); + let userId = req.query.userId; + let done = true; + + if(userId){ + + let queryResult = db.query(`UPDATE us_user u SET u.is_tutorial_complete = $1 WHERE u.uuid = $2`,[done, userId]); + queryResult + .then(result => { res.send(result);}) + .catch((err) => { res.send(err);}); + } + else{ + res.send({status : "error"}); + } + }); + + + +router.delete('/deleteScene', function(req, res, next) { + let queryResult = userStudy.deleteSceneById(req.body); + queryResult.then(result => { + // res.status(200).send('Scene deleted successfully', result); + res.status(200).send({msg:'Scene deleted successfully'}); + }).catch((err) => { + res.status(500).send(err); + console.log(err) + }); +}); + +router.get('/getAllQuestions', function (req, res) { + let userId = req.query.userId; + console.log({userId}); + let queryResult = questionUserStudy4.getQuestionByUserId(userId); + console.log({queryResult}); + queryResult + .then(result => { + res.send(result); + }) + .catch((err) => { + console.log(err); + res.send(err); + }); +}); + + +router.post('/addQuestion', function(req, res, next) { + let questionId = uuidv1(); + req.body["questionId"] = questionId; + console.log(req.body); + let queryResult = questionUserStudy4.addQuestion(req.body); + queryResult.then(result => { + console.log(result); + res.send({questionId, result : 'Question inserted successfully'}); + }).catch((err) => { + res.send(err); + console.log(err) + }); +}); + +router.post('/addCaption', function(req, res, next) { + let queryResult = questionUserStudy4.addCaption(req.body); + queryResult.then(result => { + res.send('Caption inserted successfully'); + }).catch((err) => { + res.send(err); + console.log(err) + }); +}); + +router.post('/addAnswer', function(req, res, next) { + let queryResult = questionUserStudy4.addAnswerToQuestion(req.body); + queryResult.then(result => { + res.send('Answer inserted successfully'); + }).catch((err) => { + res.send(err); + console.log(err) + }); +}); + +router.post('/updateQuestion', function(req, res, next) { + let queryResult = questionUserStudy4.updateQuestion(req.body); + queryResult.then(result => { + res.send('Question updated successfully'); + }).catch((err) => { + res.send(err); + console.log(err) + }); +}); + +router.delete('/deleteQuestion', function(req, res, next) { + let queryResult = questionUserStudy4.deleteAllQuestionsByUserId(req.body.userId); + queryResult.then(result => { + res.status(200).send('Question deleted successfully'); + }).catch((err) => { + res.status(400).send(err); + console.log(err) + }); +}); + +router.post('/addVote', function(req, res, next) { + let queryResult = questionUserStudy4.addVote(req.body); + queryResult.then(result => { + res.send('Vote inserted successfully'); + }).catch((err) => { + res.send(err); + console.log(err) + }); +}); + +module.exports = router; + + diff --git a/us3_addDescriptions.js b/us3_addDescriptions.js new file mode 100644 index 0000000..9c03a7f --- /dev/null +++ b/us3_addDescriptions.js @@ -0,0 +1,104 @@ + +const db = require('./db/database'); +const fs = require('fs'); +const csv = require('csv-parser'); +const fetch = require('node-fetch'); +const data = require('./MIvault.json'); + + + + +class us3_addDialog { + + insertIntoDb(scene_id,start_time,end_time){ + + let sqlQuery = this.createInsertStatement(scene_id,start_time,end_time) + this.addDialogTimestamps(sqlQuery); + } + + createInsertStatement(video_id,start_time, end_time){ + return `INSERT INTO video_dialogs_timestamps(video_id, start_time, end_time) VALUES('${video_id}',${start_time}, ${end_time});` + } + + addDialogTimestamps(sql, callback) { + console.log({sql}); + sql = `START TRANSACTION;` + sql + `COMMIT;`; + console.log({sql}); + let queryResult = db.query(sql, callback); + queryResult.then(result => { + console.log(result); + }).catch(e => { + console.log(e); + }); + } + } +// video_id for Ocean8 shoplifting scene + // const video_id = 'OkBaZLq7gnU'; +// video_id for MI vault scene + const video_id = '2wwC9c3iYK4'; +// video_id for notebook scene + // const video_id = '7QZNpS0uos4'; +// video_id for Mi rogue nation + // const video_id = 'elpUGB9Ap1Y'; + + let addDialog = new us3_addDialog(); + let flag = false; + let dialog_start_time; + let dialog_end_time; + let new_dialog_end_time; + // for( let val of data.result.sentence) + + let start_times = [] // Array for all start times + let end_times = [] // Array for all end times + + for(let i=0;i= 1) + start_times.push(data.result.sentence[i].startTime); + + if(i==data.result.sentence.length-1) + end_times.push(data.result.sentence[i].endTime); + else if(data.result.sentence[i+1].startTime - data.result.sentence[i].endTime >= 1) + end_times.push(data.result.sentence[i].endTime); + + } + + + + console.log("The length of start time sand end times are: "+ start_times.length +" and "+ end_times.length); + + for(let i=0; i { +// jsonFile = response.json(); +// }) + diff --git a/views/audioDescription.ejs b/views/audioDescription.ejs new file mode 100644 index 0000000..223938d --- /dev/null +++ b/views/audioDescription.ejs @@ -0,0 +1,74 @@ + + + + <%= title %> + + + + + + + + + + + + +
    +
    +
    +
    +
    Video Title Here
    +
    +
    +
    +
    +
    + + 0:00/0:00 +
    +
    + Auto-Pause at each scene: + + + +
    +
    +
    +
    + + + +
    +
    +
    + +
    +
    +
      + +
    +
    +
    +
    _
    +
    + +
    +
    +
    + + + + + \ No newline at end of file diff --git a/views/audioDescriptionUserStudy.ejs b/views/audioDescriptionUserStudy.ejs new file mode 100644 index 0000000..30290e6 --- /dev/null +++ b/views/audioDescriptionUserStudy.ejs @@ -0,0 +1,79 @@ + + + + <%= title %> + + + + + + + + + + + + +
    +
    +
    +
    +
    +
    +
    + When you watch this video, you will only hear baseline descriptions. Baseline descriptions play automatically, provide general information about a scene, and read any onscreen text out loud. Onscreen text will be read in a female voice. Press the spacebar to start and stop the video. +
    +
    +
    +
    +
    +
    +
    Video Title Here
    +
    +
    +
    +
    +
    + + 0:00/0:00 +
    +
    + + Auto-Pause at each scene: + + + + +
    +
    +
    +
    + +
    +
    +
      + +
    +
    +
    +
    +
    + + + + + + \ No newline at end of file diff --git a/views/chatbot.ejs b/views/chatbot.ejs new file mode 100644 index 0000000..1f0a929 --- /dev/null +++ b/views/chatbot.ejs @@ -0,0 +1,202 @@ + + + +

    Time taken (sec)

    +

    TIme

    + + + + + \ No newline at end of file diff --git a/views/homeAudioDescription.ejs b/views/homeAudioDescription.ejs new file mode 100644 index 0000000..9fc9076 --- /dev/null +++ b/views/homeAudioDescription.ejs @@ -0,0 +1,71 @@ + + + + User Study + + + + + + + + + +
    +
    +
    +
    +
    +

    About the Study

    +
    +

    We are conducting a research study on video description for blind and visually impaired users. Today you will be participating in a study, which should take approximately two hours. If at any point you decide you no longer wish to participate, please say so and we will stop the study and your data will not be used for the study. Responses will be confidential and completely anonymous, and your name will not appear anywhere in the final write up. You will be assigned a unique number for identification.

    + You will be describing two short videos with text. Describing a video means describing what you see on the screen in text. This includes the people, the scenes, and any text that appears on the screen. You will first be given a short video tutorial to show you how to use the video description software.

    + One of the videos you will be describing will have some text already pre-written, but the other video will not have pre-written text in the software. + Once you have described each video, we will ask you to fill out a short survey. + At the end, we will ask you a few questions about your experience. +

    +
    +
    +
    +
    +
    +

    Instructions and Tutorial

    +
    + +
    +
      + + +
    +
    +
    +
    +
    +
    +
    +

    Participant List

    +
    +

    Please click on the Participant Number assigned to you

    +
    +
    +
    +
    +
    +
    + +
    + + + \ No newline at end of file diff --git a/views/onDemand.ejs b/views/onDemand.ejs new file mode 100644 index 0000000..265dbfa --- /dev/null +++ b/views/onDemand.ejs @@ -0,0 +1,97 @@ + + + + <%= title %> + + + + + + + + + + + + +
    +
    +
    +
    +
    Video Title Here
    +
    +
    +
    +
    +
    + + 0:00/0:00 +
    +
    + Auto-Pause at each scene: + + + + +
    +

    +
    +
    + +
    +
    +
    +
    + +
    +
    +
    + + + +
    +
    +
    + +
    +
    +
      + +
    +
    +
    +
    _
    +
    + +
    +
    +
    + + + + + + + + + \ No newline at end of file diff --git a/views/onDemandTutorial.ejs b/views/onDemandTutorial.ejs new file mode 100644 index 0000000..31eda2a --- /dev/null +++ b/views/onDemandTutorial.ejs @@ -0,0 +1,241 @@ + + + + <%= title %> + + + + + + + + + + + +
    + +
    +
    +
    +
    +
    +
    +
    +
    + Be sure you are using the Google Chrome browser for all testing + activities - Remember to allow microphone access if prompted by + the Google Chrome browser. +
    + This tutorial will introduce you to two types of video + descriptions: +
    +
    +
      +
    1. + Baseline descriptions = Baseline descriptions play + automatically, provide general information about a scene, + and read any onscreen text out loud. +
      2. +
      +
    2. + Infobot = You can use the Infobot to ask for description at + any time during a video. Activate the InfoBot to get two + types of information: Click “d” at any time to get a + description of the current scene, or click “q” to ask a + question about the current scene. After each description, + you may press the up or down arrow keys to give the + description a thumbs up or down. +
      3. +
    + + This tutorial will show you how to navigate both types of + description while watching a video. +
    +
    +
    + Some screen readers use the Caps Lock key as a modifier for a + variety of screen reader commands. +
    + If any of the key commands do not seem to work, pressing Caps + Lock+Space Bar will activate form mode. Other common shortcuts + include Insert + Spacebar, Insert + Z, and Caps Lock + Z. This + should allow the key commands to work as intended. If you are + still having trouble, please look to your screen reader's + command list for a form or input. +
    + +
    + Press “D” at any time to pause the video and get an image + description of the current scene. Optionally after the response, + click the up or down arrow to give the description a thumbs up + (a chime will sound) or thumbs down (a buzzer will sound). The + video will automatically resume playing. +
    + +
    + Press Q at any time to pause the video and ask questions about + the current scene. Press Q, then wait for a short beep before + asking a question about the current scene. You should receive an + answer within 5 seconds. Optionally after the response, click + the up or down arrow to give the description a thumbs up (a + chime will sound) or thumbs down (a buzzer will sound). Ask as + many questions as you like by pressing Q and asking a question + after the beep - Only one question per beep, please! +
    +
    +
    +
    +
    + Press I to start the tutorial +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + + 0:00/0:00 +
    +
    + + Auto-Pause at each scene: + + + + + +
    + + + +
    +
    +
    + + +
    + +
    +
    +
    +
    +
    +
      +
      +
      +
      _
      +
      + +
      +
      +
      +
      +
      +
      + + + + + + + + diff --git a/views/onDemandTutorial.ejs.bak b/views/onDemandTutorial.ejs.bak new file mode 100644 index 0000000..14bbeae --- /dev/null +++ b/views/onDemandTutorial.ejs.bak @@ -0,0 +1,118 @@ + + + + <%= title %> + + + + + + + + + + + +
      + + +
      +
      +
      +
      +
      +
      +
      +

      Instructions

      +
      + +
      This tutorial will help you become acquainted with the Visual Chatbot controls.

      + When you first land on a video page, the video will initially be paused. Pressing the space bar will begin playback. Using the right and left arrow keys will scan forward and backward, respectively. Pressing the space bar again will pause playback. +

      + To ask a question, press the Q key. After the description is read, you will hear a short beep, then you may ask your question. You may ask multiple questions while the video is paused. Please try to ask questions every 30 seconds during the tutorial to help calibrate the Visual Chatbot. + +
      +
      +
      +
      +
      +
      Press Space Bar or Click Play Button to Start the Video
      +
      +
      Video Title Here
      +
      +
      +
      +
      +
      +
      +
      + + 0:00/0:00 +
      +
      + + Auto-Pause at each scene: + + + + + +
      + + +
      +
      +
      + + + +
      + +
      +
      +
      + +
      +
      +
        + +
      +
      +
      +
      _
      +
      + +
      +
      +
      + +
      + +
      +
      + + + + + + + + + \ No newline at end of file diff --git a/views/onDemandUserstudy.ejs b/views/onDemandUserstudy.ejs new file mode 100644 index 0000000..20ca176 --- /dev/null +++ b/views/onDemandUserstudy.ejs @@ -0,0 +1,96 @@ + + + + <%= title %> + + + + + + + + + + + + +
      +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      Video Title Here
      +
      +
      +
      +
      +
      + + 0:00/0:00 +
      +
      + + Auto-Pause at each scene: + + + + + +
      +
      + +
      +
      + + + +
      +
      +
      + +
      +
      +
      +
      +
        + +
      +
      +
      +
      _
      +
      +
      +
      +
      +
      +
      + + + + + + + + + \ No newline at end of file diff --git a/views/playvideodescription.ejs b/views/playvideodescription.ejs index 54ee93b..2b70fa3 100644 --- a/views/playvideodescription.ejs +++ b/views/playvideodescription.ejs @@ -1,31 +1,61 @@ - + YouDescribe - - - - - + + + + + - - - + + + -
      +
      -
      -
      -
      Video Title Here
      -
      -
      - -
      -
      - -
      +
      +
      - - + + diff --git a/views/uploadVideo.ejs b/views/uploadVideo.ejs new file mode 100644 index 0000000..1b7e717 --- /dev/null +++ b/views/uploadVideo.ejs @@ -0,0 +1,92 @@ + + + User Study + + + + + + + + + + + +
      + +
      +
      + + +
      +
      +
      +
      + +
      +
      + + + Please provide the youtube id of the video. This id will be used to play the video from the youtube +
      +
      + + +
      +
      + + +
      +
      +
      +
      + + + + +
      +
      + +
      +
      + + + +
      +

      Insert the video Insight into Database

      + +
      +
      +
      +
      +
      +
      + Audio File:
      + Your Callback Url:
      + + +
      +
      +
      + +
      +
      + + + + +
      +
      +
      +

      Merge scenes based on audio timestamps

      + +
      +
      + + + + + + \ No newline at end of file diff --git a/views/userStudy4/homeUserStudy4.ejs b/views/userStudy4/homeUserStudy4.ejs new file mode 100644 index 0000000..2a3315b --- /dev/null +++ b/views/userStudy4/homeUserStudy4.ejs @@ -0,0 +1,27 @@ + + + + User Study 4 + + + + + + + + +
      +
      +
      +

      Participant List

      +
      +

      Please click on the Participant Number assigned to you

      +
      +
      +
      +
      +
      + + + + \ No newline at end of file diff --git a/views/userStudy4/videoUserStudy4.ejs b/views/userStudy4/videoUserStudy4.ejs new file mode 100644 index 0000000..e21c860 --- /dev/null +++ b/views/userStudy4/videoUserStudy4.ejs @@ -0,0 +1,30 @@ + + + + User Study + + + + + + + + + +
      +
      + Note: Please do the tutorial first. Please ensure that you attempt the video descriptions in sequential order only (i.e, Video 1 Before Video 2) +
      +
      +
      + +
      +
      + + + \ No newline at end of file diff --git a/views/videoHelper.ejs b/views/videoHelper.ejs index de16dd3..efc2f56 100644 --- a/views/videoHelper.ejs +++ b/views/videoHelper.ejs @@ -58,7 +58,6 @@ let url = location.search.substring(1); let pid = new URLSearchParams(url).get('pid'); let num_videos = (pid % 2 == 0) ? 20 : 22; let num_describers = (pid % 2 == 0) ? 10 : 11; - let intro = `Thank you for participating in this user study. Please answer the questions below, and then watch the videos, and answer the questions about each video. There are ${num_videos} videos, and each video is about 2 minutes long. The ${num_videos} videos are in fact only 2 different videos described by ${num_describers} different describers. Even though they are repetitive, please listen to each description, rate the description quality, and leave comments for each video. @@ -69,27 +68,21 @@ $("#intro").append(intro); @@ -332,5 +299,4 @@ function fetchVideo() { - - \ No newline at end of file + \ No newline at end of file diff --git a/views/videoaudio.ejs b/views/videoaudio.ejs index 339fe27..1e57484 100644 --- a/views/videoaudio.ejs +++ b/views/videoaudio.ejs @@ -1,5 +1,5 @@ - + YouDescribe @@ -7,15 +7,15 @@ - + - +
      - + \ No newline at end of file diff --git a/views/withoutai.ejs b/views/withoutai.ejs index 3dd81c5..ee3d1af 100644 --- a/views/withoutai.ejs +++ b/views/withoutai.ejs @@ -75,7 +75,7 @@ }); - + \ No newline at end of file diff --git a/youdescribe-8f42e2133126.json b/youdescribe-8f42e2133126.json new file mode 100644 index 0000000..4adf13d --- /dev/null +++ b/youdescribe-8f42e2133126.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "youdescribe-250406", + "private_key_id": "8f42e213312606e5d8508b22da34fbf63f8cdf0c", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDHiZHsdVIOZ9r9\n+BOmKEidzERC87GdEmUrKKHU5A6BW/ZdQPDmgspvnsX3T28lj0tyNdI0Qtf2l9pV\nPxYxIk488+FwRf8gonOAs+EA2jW//7eBes5JR2Sy51YyYCGdEyStJua0k+gAGEVe\nlOlYoJQnM1bsIOllsJEGy3MvO+xZAEBni3/iXT1f2fAE22qlth+Fmm/rdnoV5tOA\n6RyHkn7uWN3HgznRPrsvKDZekouzK9+aF6tnTruVNV1b2U2vTiFuVsGIjF59vEt2\nOnnkKsBti6E1kQDXyBDFpX7TaBiGHcZlIG/Hs65B2XucsDSOlTAXUTuiOgjc892q\nTZCvCWNBAgMBAAECggEALD8UxSbk2b3s4TlT/hnbtGYZ0vDf91EfbvkVHbSuhL/k\nOxyN7sec5TCDfIo4Ps9uLvbBVIZoBk35A8M4BBAz4i9AU9mpr85mJ2l74OkCVz94\nrTUSrkwvB77lrckEWyTmEqFYMj3ECdVX+JezwMvm3mRzh2vq5XpIRoXx1k0G5VsV\nJD4MePzAHXBVq92n1Dab7m8PPtLJuungfi3d0xyp4Z40RCbRovpDMa+TwUbVbFJe\nb0nEKx4AuamcCXtz78DBEnJuROk4OWeYN0a2YvlKPoYug7I1LDQ5osYQqupoViBn\nM/dMX3B/ydCd1o8+DmkDvUGKst6hNI6KN7Pv2I2mUwKBgQDpJ03wp+72f+SdqlMf\nkB66GXptK9SgM6dkFVIN6xs3+iazXjvxlwPDAL19Rih/38KaWjqsUbLZpUkyyfXL\nZdg+bf5NZz+gI62P8DzziySqN7ntFmzcTdIkR6ArRpJVFpknCDCyyRbjpfIerAty\n6X8zA5AftxpX5iaYf05ZDkcpnwKBgQDbFv+wfGyMeJlf6ghopVGisGCPbfTdNaf8\nak59OrL1kH+bNQCcmfSuCayx4CFsB/rnWKUexu8p2/DAbMgMKUy40Slu/TSXDsw2\nqwvTbAKoG0zPZtVHk/xbOXmAvBZ4X+ODLiPvtvsDhQ7RymPC5JSUwtcsasdmbcYd\nBr266LgHHwKBgQDBAn2eBJdozjUGZFLNpTQaoS7dI1SwjjI7Qv8bcI6mx1m0UQVU\nmMxPaNR02Vh7i1QBDULXFgb178Np3RCyIYxjGhFUKG3edsqehqUehAiFNP6gi1jx\nAyc8S2MHkDNhZwS/8CLITnm5WM3XoFsV9MANDh0IdD7Fl0xKH9FPHefBRwKBgQDP\n5p0v7oJbMj3pBH6CPw6eoVyf/qm0GKyWHJLqxv73g+1zbPJtqLLFTCzyhktw5omz\nay0VbZWaVTm6f/+9eptxbhoxVlnHVqV24w1PzVkgCPf0gBK337hDIOEpxlJUXDnb\nS7eSTEMabfNIJRRR5vMPz2y8m38otdafyLvWV+v55QKBgCzC1Ssv2RF33p5ltYLs\nrWVon5IQxt/Ol0DJIkLVAEQT4T5bBhT3JeEhKGjHI8WsGsSvdTkV4qgUXsEb0lD1\njpqUqks0GWd3dbSAyCzMqjHbkT97ovIwc+b83K940KXoim8oOQW6AtVtG86K8KZZ\nfD6Za1P/nWY39RdJDK3crsbO\n-----END PRIVATE KEY-----\n", + "client_email": "youdescribe-service-account@youdescribe-250406.iam.gserviceaccount.com", + "client_id": "114749674331828316527", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/youdescribe-service-account%40youdescribe-250406.iam.gserviceaccount.com" +}