diff --git a/package.json b/package.json index ac883585..152bbbdc 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "typescript": "^4.1.5", "typescript-is": "^0.18.2", "uuid": "^8.3.2", + "vdata-parser": "^0.1.5", "yauzl": "^2.10.0" }, "devDependencies": { diff --git a/src/classes/Standardizer/plugins/Discord/getters/getTransactionsStatistics.ts b/src/classes/Standardizer/plugins/Discord/getters/getTransactionsStatistics.ts new file mode 100644 index 00000000..ed96dbb9 --- /dev/null +++ b/src/classes/Standardizer/plugins/Discord/getters/getTransactionsStatistics.ts @@ -0,0 +1,35 @@ +import Discord from '../Discord' +import Calculator from '../../statClasses/Transaction'; +import { StatisticType } from '../../../../../types/schemas/Statistic'; + +Discord.prototype.getTransactionsStatistics = async function getTransactionsStatistics() { + const calculator = new Calculator() + let chunk + const size = 50 + let offset = 0 + do { + // eslint-disable-next-line no-await-in-loop + chunk = await this.getTransactions({ + parsingOptions: { + pagination: { + offset, + items: size, + }, + }, + }) + calculator.process_record(chunk?.data ?? []) + offset += size + } while (chunk?.data?.length === 50) + + return { + statistics: [ + calculator.total, + { + type: StatisticType.RANKING, + name: 'AverageMonth', + value: await calculator.averages, + }, + ], + parsedFiles: chunk?.parsedFiles ?? [], + } +} diff --git a/src/classes/Standardizer/plugins/Facebook/getters/getNotificationsStatistics.ts b/src/classes/Standardizer/plugins/Facebook/getters/getNotificationsStatistics.ts new file mode 100644 index 00000000..70751e5a --- /dev/null +++ b/src/classes/Standardizer/plugins/Facebook/getters/getNotificationsStatistics.ts @@ -0,0 +1,35 @@ +import Facebook from '../Facebook' +import Calculator from '../../statClasses/Notification'; +import { StatisticType } from '../../../../../types/schemas/Statistic' + +Facebook.prototype.getNotificationsStatistics = async function getNotificationsStatistics() { + const calculator = new Calculator() + let chunk + const size = 50 + let offset = 0 + do { + // eslint-disable-next-line no-await-in-loop + chunk = await this.getNotifications({ + parsingOptions: { + pagination: { + offset, + items: size, + }, + }, + }) + calculator.process_record(chunk?.data ?? []) + offset += size + } while (chunk?.data?.length === 50) + + return { + statistics: [ + { + name: 'Daily Average', + type: StatisticType.NUMBER, + value: await calculator.average, + description: 'average number of notifications per day', + }, + ], + parsedFiles: chunk?.parsedFiles ?? [], + } +} diff --git a/src/classes/Standardizer/plugins/Google/getters/getConnectionsStatistics.ts b/src/classes/Standardizer/plugins/Google/getters/getConnectionsStatistics.ts new file mode 100644 index 00000000..6cda588b --- /dev/null +++ b/src/classes/Standardizer/plugins/Google/getters/getConnectionsStatistics.ts @@ -0,0 +1,45 @@ +import Google from '../Google' +import Calculator from '../../statClasses/Connection'; +import { StatisticType } from '../../../../../types/schemas/Statistic' + +Google.prototype.getConnectionsStatistics = async function getConnectionsStatistics() { + const calculator = new Calculator() + let chunk + const size = 50 + let offset = 0 + do { + // eslint-disable-next-line no-await-in-loop + chunk = await this.getConnections({ + parsingOptions: { + pagination: { + offset, + items: size, + }, + }, + }) + calculator.process_record(chunk?.data ?? []) + offset += size + } while (chunk?.data?.length === 50) + + return { + statistics: [ + { + name: 'Monthly Average', + type: StatisticType.NUMBER, + value: await calculator.average, + description: 'average number of connections per month', + }, + { + name: 'Top IPs', + type: StatisticType.RANKING, + value: calculator.ips, + }, + { + name: 'Top locations', + type: StatisticType.RANKING, + value: calculator.locations, + }, + ], + parsedFiles: chunk?.parsedFiles ?? [], + } +} diff --git a/src/classes/Standardizer/plugins/Google/getters/getTasksStatistics.ts b/src/classes/Standardizer/plugins/Google/getters/getTasksStatistics.ts new file mode 100644 index 00000000..2eea168c --- /dev/null +++ b/src/classes/Standardizer/plugins/Google/getters/getTasksStatistics.ts @@ -0,0 +1,41 @@ +import Google from '../Google' +import Calculator from '../../statClasses/Task'; +import { StatisticType } from '../../../../../types/schemas/Statistic' + +Google.prototype.getTasksStatistics = async function getTasksStatistics() { + const calculator = new Calculator() + let chunk + const size = 50 + let offset = 0 + do { + // eslint-disable-next-line no-await-in-loop + chunk = await this.getTasks({ + parsingOptions: { + pagination: { + offset, + items: size, + }, + }, + }) + calculator.process_record(chunk?.data ?? []) + offset += size + } while (chunk?.data?.length === 50) + + return { + statistics: [ + { + name: 'Monthly Average', + type: StatisticType.NUMBER, + value: await calculator.average, + description: 'average number of tasks created per Month', + }, + { + name: 'Tasks Done', + type: StatisticType.PERCENTAGE, + value: calculator.percentDone, + description: 'percentage of tasks done', + }, + ], + parsedFiles: chunk?.parsedFiles ?? [], + } +} diff --git a/src/classes/Standardizer/plugins/Google/getters/getTransactionsStatistics.ts b/src/classes/Standardizer/plugins/Google/getters/getTransactionsStatistics.ts new file mode 100644 index 00000000..b06769bb --- /dev/null +++ b/src/classes/Standardizer/plugins/Google/getters/getTransactionsStatistics.ts @@ -0,0 +1,35 @@ +import Google from '../Google' +import Calculator from '../../statClasses/Transaction'; +import { StatisticType } from '../../../../../types/schemas/Statistic'; + +Google.prototype.getTransactionsStatistics = async function getTransactionsStatistics() { + const calculator = new Calculator() + let chunk + const size = 50 + let offset = 0 + do { + // eslint-disable-next-line no-await-in-loop + chunk = await this.getTransactions({ + parsingOptions: { + pagination: { + offset, + items: size, + }, + }, + }) + calculator.process_record(chunk?.data ?? []) + offset += size + } while (chunk?.data?.length === 50) + + return { + statistics: [ + calculator.total, + { + type: StatisticType.RANKING, + name: 'AverageMonth', + value: await calculator.averages, + }, + ], + parsedFiles: chunk?.parsedFiles ?? [], + } +} diff --git a/src/classes/Standardizer/plugins/Reddit/getters/getTransactionsStatistics.ts b/src/classes/Standardizer/plugins/Reddit/getters/getTransactionsStatistics.ts new file mode 100644 index 00000000..25e9b484 --- /dev/null +++ b/src/classes/Standardizer/plugins/Reddit/getters/getTransactionsStatistics.ts @@ -0,0 +1,35 @@ +import Reddit from '../Reddit' +import Calculator from '../../statClasses/Transaction'; +import { StatisticType } from '../../../../../types/schemas/Statistic'; + +Reddit.prototype.getTransactionsStatistics = async function getTransactionsStatistics() { + const calculator = new Calculator() + let chunk + const size = 50 + let offset = 0 + do { + // eslint-disable-next-line no-await-in-loop + chunk = await this.getTransactions({ + parsingOptions: { + pagination: { + offset, + items: size, + }, + }, + }) + calculator.process_record(chunk?.data ?? []) + offset += size + } while (chunk?.data?.length === 50) + + return { + statistics: [ + calculator.total, + { + type: StatisticType.RANKING, + name: 'AverageMonth', + value: await calculator.averages, + }, + ], + parsedFiles: chunk?.parsedFiles ?? [], + } +} diff --git a/src/classes/Standardizer/plugins/statClasses/Connection.ts b/src/classes/Standardizer/plugins/statClasses/Connection.ts new file mode 100644 index 00000000..6f4a908e --- /dev/null +++ b/src/classes/Standardizer/plugins/statClasses/Connection.ts @@ -0,0 +1,92 @@ +import moment from 'moment'; +import { Connection } from '../../../../types/schemas'; + +export default class Calculator { + constructor() { + this.calendar = {} + this.ip = {} + this.location = {} + } + + process_record(chunk: Array) { + this.updateAverage(chunk) + this.updateRanking(chunk) + } + + private readonly calendar: Record + + private readonly ip: Record + + private readonly location: Record + + private updateRanking(chunk: Array) { + chunk.forEach(connection => { + // eslint-disable-next-line no-prototype-builtins + if (!this.ip.hasOwnProperty(connection.ipAddress)) { + this.ip[connection.ipAddress] = 1 + } else { + this.ip[connection.ipAddress] += 1 + } + const location = connection.location?.relativePosition?.city + if (!location) return + // eslint-disable-next-line no-prototype-builtins + if (!this.location.hasOwnProperty(location)) { + this.location[location] = 1 + } else { + this.ip[location] += 1 + } + }) + } + + private sort(obj: Object) { + return Object.fromEntries( + Object.entries(obj).sort(([,a], [,b]) => b - a), + ); + } + + get ips() { + return Object.entries(this.sort(this.ip)).map(([ip, nb]) => ({ + label: ip, + value: nb, + })).slice(0, 5) + } + + get locations() { + return Object.entries(this.sort(this.location)).map(([location, nb]) => ({ + label: location, + value: nb, + })).slice(0, 5) + } + + private updateAverage(chunk: Array) { + /** + * calendar = { + * '09-2021': [0,100,...] + * ... + * } + */ + chunk.forEach(connection => { + const { calendar } = this + const date = moment(connection.timestamp) + // eslint-disable-next-line no-prototype-builtins + if (!calendar.hasOwnProperty(date.format('YYYY-MM'))) { + calendar[date.format('YYYY-MM')] = Array.from(new Array(date.daysInMonth()), () => 0) + } + calendar[date.format('YYYY-MM')][date.date() - 1] += 1 + }) + } + + get average() { + return this.computeAverage() + } + + private async computeAverage() { + const { calendar } = this + let sum: Array = [] + await Object.keys(calendar).forEach(month => { + sum = sum.concat(calendar[month]) + }) + const nb = sum.length + return sum.reduce((a, b) => a + b, 0) / nb + } +} diff --git a/src/classes/Standardizer/plugins/statClasses/Notification.ts b/src/classes/Standardizer/plugins/statClasses/Notification.ts new file mode 100644 index 00000000..33b75362 --- /dev/null +++ b/src/classes/Standardizer/plugins/statClasses/Notification.ts @@ -0,0 +1,46 @@ +import moment from 'moment'; +import { Notification } from '../../../../types/schemas'; + +export default class Calculator { + constructor() { + this.calendar = {} + } + + process_record(chunk: Array) { + this.updateAverage(chunk) + } + + private readonly calendar: Record + + private updateAverage(chunk: Array) { + /** + * calendar = { + * '09-2021': [0,100,...] + * ... + * } + */ + chunk.forEach(notification => { + const { calendar } = this + const date = moment(notification.notificationDate) + // eslint-disable-next-line no-prototype-builtins + if (!calendar.hasOwnProperty(date.format('YYYY-MM'))) { + calendar[date.format('YYYY-MM')] = Array.from(new Array(date.daysInMonth()), () => 0) + } + calendar[date.format('YYYY-MM')][date.date() - 1] += 1 + }) + } + + get average() { + return this.computeAverage() + } + + private async computeAverage() { + const { calendar } = this + let sum: Array = [] + await Object.keys(calendar).forEach(month => { + sum = sum.concat(calendar[month]) + }) + const nb = sum.length + return sum.reduce((a, b) => a + b, 0) / nb + } +} diff --git a/src/classes/Standardizer/plugins/statClasses/Task.ts b/src/classes/Standardizer/plugins/statClasses/Task.ts new file mode 100644 index 00000000..4038b668 --- /dev/null +++ b/src/classes/Standardizer/plugins/statClasses/Task.ts @@ -0,0 +1,68 @@ +import moment from 'moment'; +import { TaskList } from '../../../../types/schemas'; + +export default class Calculator { + constructor() { + this.calendar = {} + this.total = 0 + this.done = 0 + } + + process_record(chunk: Array) { + this.updateAverage(chunk) + this.updateTotal(chunk) + } + + private readonly calendar: Record + + private total: number + + private done: number + + private updateTotal(chunk: Array) { + chunk.forEach(taskList => { + taskList.tasks.forEach(task => { + this.total += 1 + this.done += task.status === 'done' ? 1 : 0 + }) + }) + } + + private updateAverage(chunk: Array) { + /** + * calendar = { + * '09-2021': [0,100,...] + * ... + * } + */ + chunk.forEach(taskList => { + taskList.tasks.forEach(task => { + const { calendar } = this + const date = moment(task.createdAt) + // eslint-disable-next-line no-prototype-builtins + if (!calendar.hasOwnProperty(date.format('YYYY-MM'))) { + calendar[date.format('YYYY-MM')] = Array.from(new Array(date.daysInMonth()), () => 0) + } + calendar[date.format('YYYY-MM')][date.date() - 1] += 1 + }) + }) + } + + get average() { + return this.computeAverage() + } + + get percentDone() { + return (this.done * 100) / this.total + } + + private async computeAverage() { + const { calendar } = this + const sum: Array = [] + await Object.keys(calendar).forEach(month => { + sum.push(calendar[month].reduce((a: number, b: number) => a + b, 0)) + }) + const nb = sum.length + return sum.reduce((a, b) => a + b, 0) / nb + } +} diff --git a/src/classes/Standardizer/plugins/statClasses/Transaction.ts b/src/classes/Standardizer/plugins/statClasses/Transaction.ts new file mode 100644 index 00000000..7611d777 --- /dev/null +++ b/src/classes/Standardizer/plugins/statClasses/Transaction.ts @@ -0,0 +1,99 @@ +import moment from 'moment'; +import { Transaction } from '../../../../types/schemas'; +import { StatisticType } from '../../../../types/schemas/Statistic'; + +interface Amounts { + [key: string]: number +} + +export default class Calculator { + private readonly spent: Amounts + + private currency: Array + + constructor() { + this.currency = [] + this.spent = {} + this.calendars = [] + } + + process_record(chunk: Array) { + this.updateSpent(chunk) + this.updateAverage(chunk) + } + + private updateSpent(chunk: Array) { + chunk.forEach((item) => { + // eslint-disable-next-line no-prototype-builtins + if (!this.spent.hasOwnProperty(item.currency)) this.spent[item.currency] = 0 + this.spent[item.currency] += item?.value ?? 0 + }) + } + + private readonly calendars: Array> + + private updateAverage(chunk: Array) { + /** + * calendar = { + * '09-2021': [[], [0,100], [],...] + * ... + * } + */ + chunk.forEach(transaction => { + if (this.currency.indexOf(transaction.currency) === -1) { + this.currency.push(transaction.currency) + this.calendars.push({}) + } + const calendar = this.calendars[this.currency.indexOf(transaction.currency)] + const date = moment(transaction.date) + // eslint-disable-next-line no-prototype-builtins + if (!calendar.hasOwnProperty(date.format('YYYY-MM'))) { + calendar[date.format('YYYY-MM')] = Array.from(new Array(date.daysInMonth()), () => []) + } + calendar[date.format('YYYY-MM')][date.date() - 1].push(transaction.value) + }) + } + + get total() { + return { + type: StatisticType.RANKING, + name: 'total', + value: Object.keys(this.spent).map(currency => ({ + label: currency, + value: this.spent[currency], + })), + } + } + + get averages() { + return this.buildStat() + } + + private async buildStat() { + const average = this.currency.map(async currency => ({ + label: currency, + value: await this.computeAverage(currency), + })) + return Promise.all(average) + } + + private async computeAverage(currency: string) { + /** + * calendar_processed = + * {'09-2021': 3454 //average, ...} + */ + const calendar = this.calendars[this.currency.indexOf(currency)] + let sum = 0 + await Object.keys(calendar).forEach(month => { + calendar[month] = calendar[month].map((day: Array) => { + if (!day.length) return 0 + return day.reduce((a: number, b: number) => a + b, 0) + }) + const nb = calendar[month].length + calendar[month] = calendar[month].reduce((a: number, b: number) => a + b, 0) + calendar[month] /= nb + sum += calendar[month] + }) + return sum /= Object.keys(calendar).length + } +} diff --git a/src/modules/Parsing/parseAsICS.ts b/src/modules/Parsing/parseAsICS.ts index 93035a99..8b52ac40 100644 --- a/src/modules/Parsing/parseAsICS.ts +++ b/src/modules/Parsing/parseAsICS.ts @@ -1,4 +1,6 @@ -import { ParsingOptions } from '../../types/Parsing' +// @ts-ignore +import parser from 'vdata-parser' +import { ParsingOptions, ParsingReturn } from '../../types/Parsing' import Pipeline from '../../classes/Pipeline' export type OptionsParseAsICS = ParsingOptions @@ -6,7 +8,13 @@ export type OptionsParseAsICS = ParsingOptions /** * Parse given Pipeline result stream as ICS format */ -// eslint-disable-next-line @typescript-eslint/no-unused-vars -export default async function parseAsICS(pipeline: Pipeline, options?: OptionsParseAsICS): Promise { - return Promise.reject(new Error('Not implemented')) +export default async function parseAsICS(pipeline: Pipeline, options?: OptionsParseAsICS): Promise> { + return { + pagination: { + total: 1, + items: 1, + offset: 0, + }, + data: parser.fromString(await pipeline.toString()), + } } diff --git a/src/modules/Parsing/parseAsVCARD.ts b/src/modules/Parsing/parseAsVCARD.ts index 71b9aa3f..7829cae5 100644 --- a/src/modules/Parsing/parseAsVCARD.ts +++ b/src/modules/Parsing/parseAsVCARD.ts @@ -1,3 +1,5 @@ +// @ts-ignore +import parser from 'vdata-parser' import { ParsingOptions, ParsingReturn } from '../../types/Parsing' import Pipeline from '../../classes/Pipeline' @@ -6,11 +8,14 @@ export type OptionsParseAsVCARD = ParsingOptions /** * Parse given Pipeline result stream as VCARD format */ -export default async function parseAsVCARD( - // eslint-disable-next-line @typescript-eslint/no-unused-vars - pipeline: Pipeline, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - options?: OptionsParseAsVCARD, -): Promise> { - return Promise.reject(new Error('Not implemented')) +export default async function parseAsVCARD(pipeline: Pipeline, options?: OptionsParseAsVCARD): +Promise> { + return { + pagination: { + total: 1, + items: 1, + offset: 0, + }, + data: parser.fromString(await pipeline.toString()), + } } diff --git a/yarn.lock b/yarn.lock index d0c37e20..60678f31 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4362,6 +4362,11 @@ validate-npm-package-license@^3.0.1: spdx-correct "^3.0.0" spdx-expression-parse "^3.0.0" +vdata-parser@^0.1.5: + version "0.1.5" + resolved "https://registry.yarnpkg.com/vdata-parser/-/vdata-parser-0.1.5.tgz#e57cf30065cf69d29a9d822e85fd2734c686bcfd" + integrity sha1-5XzzAGXPadKanYIuhf0nNMaGvP0= + verror@1.10.0: version "1.10.0" resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz"