issue/37 (#42)

Co-authored-by: dbroqua <contact@darkou.fr>
Reviewed-on: https://git.darkou.fr/dbroqua/MusicTopus/pulls/42
This commit is contained in:
Damien Broqua 2022-04-09 00:07:22 +02:00
parent 182aa7a6a6
commit ae4b7b6de0
14 changed files with 822 additions and 472 deletions

View file

@ -7,6 +7,8 @@ import flash from "connect-flash";
import session from "express-session";
import MongoStore from "connect-mongo";
import passportConfig from "./libs/passport";
import config, { env, mongoDbUri, secret } from "./config";
import { isXhr } from "./helpers";
@ -15,15 +17,13 @@ import indexRouter from "./routes";
import maCollectionRouter from "./routes/ma-collection";
import collectionRouter from "./routes/collection";
import importJobsRouter from "./routes/jobs";
import importAlbumRouterApiV1 from "./routes/api/v1/albums";
import importSearchRouterApiV1 from "./routes/api/v1/search";
import importMeRouterApiV1 from "./routes/api/v1/me";
// Mongoose schema init
require("./models/users");
require("./models/albums");
require("./libs/passport")(passport);
passportConfig(passport);
mongoose
.connect(mongoDbUri, { useNewUrlParser: true, useUnifiedTopology: true })
@ -46,10 +46,10 @@ const sess = {
const app = express();
app.use(express.json());
app.use(express.urlencoded({ extended: false }));
app.use(cookieParser());
app.use(flash());
app.use(express.json({ limit: "50mb" }));
app.use(express.urlencoded({ extended: false, limit: "50mb" }));
app.use(session(sess));
@ -85,6 +85,7 @@ app.use(
app.use("/", indexRouter);
app.use("/ma-collection", maCollectionRouter);
app.use("/collection", collectionRouter);
app.use("/jobs", importJobsRouter);
app.use("/api/v1/albums", importAlbumRouterApiV1);
app.use("/api/v1/search", importSearchRouterApiV1);
app.use("/api/v1/me", importMeRouterApiV1);

View file

@ -8,4 +8,13 @@ module.exports = {
matomoUrl: process.env.MATOMO_URL || "",
matomoId: process.env.MATOMO_ID || "",
siteName: process.env.SITE_NAME || "MusicTopus",
awsAccessKeyId: process.env.AWS_ACCESS_KEY_ID,
awsSecretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
s3BaseFolder: process.env.S3_BASEFOLDER || "dev",
s3Bucket: process.env.S3_BUCKET || "musictopus",
s3Endpoint: process.env.S3_ENDPOINT || "s3.fr-par.scw.cloud",
s3Signature: process.env.S3_SIGNATURE || "s3v4",
jobsHeaderKey: process.env.JOBS_HEADER_KEY || "musictopus",
jobsHeaderValue:
process.env.JOBS_HEADER_VALUE || "ooYee9xok7eigo2shiePohyoGh1eepew",
};

72
src/libs/aws.js Normal file
View file

@ -0,0 +1,72 @@
import AWS from "aws-sdk";
import fs from "fs";
import path from "path";
import axios from "axios";
import { v4 as uuid } from "uuid";
import {
awsAccessKeyId,
awsSecretAccessKey,
s3BaseFolder,
s3Endpoint,
s3Bucket,
s3Signature,
} from "../config";
AWS.config.update({
accessKeyId: awsAccessKeyId,
secretAccessKey: awsSecretAccessKey,
});
/**
* Fonction permettant de stocker un fichier local sur S3
* @param {String} filename
* @param {String} file
* @param {Boolean} deleteFile
*
* @return {String}
*/
export const uploadFromFile = async (filename, file, deleteFile = false) => {
const data = await fs.readFileSync(file);
const base64data = Buffer.from(data, "binary");
const dest = path.join(s3BaseFolder, filename);
const s3 = new AWS.S3({
endpoint: s3Endpoint,
signatureVersion: s3Signature,
});
await s3
.putObject({
Bucket: s3Bucket,
Key: dest,
Body: base64data,
ACL: "public-read",
})
.promise();
if (deleteFile) {
fs.unlinkSync(file);
}
return `https://${s3Bucket}.${s3Endpoint}/${dest}`;
};
/**
* Fonction permettant de stocker un fichier provenant d'une URL sur S3
* @param {String} url
*
* @return {String}
*/
export const uploadFromUrl = async (url) => {
const filename = `${uuid()}.jpg`;
const file = `/tmp/${filename}`;
const { data } = await axios.get(url, { responseType: "arraybuffer" });
fs.writeFileSync(file, data);
return uploadFromFile(filename, file, true);
// return s3Object;
};

View file

@ -1,11 +1,13 @@
/* eslint-disable func-names */
const mongoose = require("mongoose");
const LocalStrategy = require("passport-local").Strategy;
const { BasicStrategy } = require("passport-http");
import { Strategy as LocalStrategy } from "passport-local";
import { BasicStrategy } from "passport-http";
import { Strategy as CustomStrategy } from "passport-custom";
const Users = mongoose.model("Users");
import Users from "../models/users";
module.exports = function (passport) {
import { jobsHeaderKey, jobsHeaderValue } from "../config";
export default (passport) => {
passport.serializeUser((user, done) => {
done(null, user);
});
@ -55,4 +57,17 @@ module.exports = function (passport) {
.catch(done);
})
);
passport.use(
"jobs",
new CustomStrategy((req, next) => {
const apiKey = req.headers[jobsHeaderKey];
if (apiKey === jobsHeaderValue) {
return next(null, {
username: "jobs",
});
}
return next(null, false, "Oops! Identifiants incorrects");
})
);
};

View file

@ -1,468 +1,18 @@
import moment from "moment";
import momenttz from "moment-timezone";
import xl from "excel4node";
import Pages from "./Pages";
import Export from "./Export";
import AlbumsModel from "../models/albums";
import JobsModel from "../models/jobs";
import UsersModel from "../models/users";
import ErrorEvent from "../libs/error";
// import { uploadFromUrl } from "../libs/aws";
/**
* Classe permettant la gestion des albums d'un utilisateur
*/
class Albums extends Pages {
/**
* Méthode permettant de remplacer certains cartactères par leur équivalents html
* @param {String} str
*
* @return {String}
*/
static replaceSpecialChars(str) {
if (!str) {
return "";
}
let final = str.toString();
const find = ["&", "<", ">"];
const replace = ["&amp;", "&lt;", "&gt;"];
for (let i = 0; i < find.length; i += 1) {
final = final.replace(new RegExp(find[i], "g"), replace[i]);
}
return final;
}
/**
* Méthode permettant de convertir les rows en csv
* @param {Array} rows
*
* @return {string}
*/
static async convertToCsv(rows) {
let data =
"Artiste;Titre;Genre;Styles;Pays;Année;Date de sortie;Format\n\r";
for (let i = 0; i < rows.length; i += 1) {
const {
artists_sort,
title,
genres,
styles,
country,
year,
released,
formats,
} = rows[i];
let format = "";
for (let j = 0; j < formats.length; j += 1) {
format += `${format !== "" ? ", " : ""}${formats[j].name}`;
}
data += `${artists_sort};${title};${genres.join()};${styles.join()};${country};${year};${released};${format}\n\r`;
}
return data;
}
/**
* Méthode permettant de convertir les rows en fichier xls
* @param {Array} rows
*
* @return {Object}
*/
static async convertToXls(rows) {
const wb = new xl.Workbook();
const ws = wb.addWorksheet("MusicTopus");
const headerStyle = wb.createStyle({
font: {
color: "#FFFFFF",
size: 11,
},
fill: {
type: "pattern",
patternType: "solid",
bgColor: "#595959",
fgColor: "#595959",
},
});
const style = wb.createStyle({
font: {
color: "#000000",
size: 11,
},
numberFormat: "0000",
});
const header = [
"Artiste",
"Titre",
"Genre",
"Styles",
"Pays",
"Année",
"Date de sortie",
"Format",
];
for (let i = 0; i < header.length; i += 1) {
ws.cell(1, i + 1)
.string(header[i])
.style(headerStyle);
}
for (let i = 0; i < rows.length; i += 1) {
const currentRow = i + 2;
const {
artists_sort,
title,
genres,
styles,
country,
year,
released,
formats,
} = rows[i];
let format = "";
for (let j = 0; j < formats.length; j += 1) {
format += `${format !== "" ? ", " : ""}${formats[j].name}`;
}
ws.cell(currentRow, 1).string(artists_sort).style(style);
ws.cell(currentRow, 2).string(title).style(style);
ws.cell(currentRow, 3).string(genres.join()).style(style);
ws.cell(currentRow, 4).string(styles.join()).style(style);
if (country) {
ws.cell(currentRow, 5).string(country).style(style);
}
if (year) {
ws.cell(currentRow, 6).number(year).style(style);
}
if (released) {
ws.cell(currentRow, 7)
.date(momenttz.tz(released, "Europe/Paris").hour(12))
.style({ numberFormat: "dd/mm/yyyy" });
}
ws.cell(currentRow, 8).string(format).style(style);
}
return wb;
}
/**
* Méthode permettant de convertir les rows en csv pour importer dans MusicTopus
* @param {Array} rows
*
* @return {string}
*/
static async convertToXml(rows) {
let data = '<?xml version="1.0" encoding="UTF-8"?>\n\r<albums>';
for (let i = 0; i < rows.length; i += 1) {
const {
discogsId,
year,
released,
uri,
artists,
artists_sort,
labels,
series,
companies,
formats,
title,
country,
notes,
identifiers,
videos,
genres,
styles,
tracklist,
extraartists,
images,
thumb,
} = rows[i];
let artistsList = "";
let labelList = "";
let serieList = "";
let companiesList = "";
let formatsList = "";
let identifiersList = "";
let videosList = "";
let genresList = "";
let stylesList = "";
let tracklistList = "";
let extraartistsList = "";
let imagesList = "";
for (let j = 0; j < artists.length; j += 1) {
artistsList += `<artist>
<name>${Albums.replaceSpecialChars(artists[j].name)}</name>
<anv>${Albums.replaceSpecialChars(artists[j].anv)}</anv>
<join>${Albums.replaceSpecialChars(artists[j].join)}</join>
<role>${Albums.replaceSpecialChars(artists[j].role)}</role>
<tracks>${Albums.replaceSpecialChars(
artists[j].tracks
)}</tracks>
<id>${Albums.replaceSpecialChars(artists[j].id)}</id>
<resource_url>${Albums.replaceSpecialChars(
artists[j].resource_url
)}</resource_url>
<thumbnail_url>${Albums.replaceSpecialChars(
artists[j].thumbnail_url
)}</thumbnail_url>
</artist>`;
}
for (let j = 0; j < labels.length; j += 1) {
labelList += `<label>
<name>${Albums.replaceSpecialChars(labels[j].name)}</name>
<catno>${Albums.replaceSpecialChars(labels[j].catno)}</catno>
<entity_type>${Albums.replaceSpecialChars(
labels[j].entity_type
)}</entity_type>
<entity_type_name>${Albums.replaceSpecialChars(
labels[j].entity_type
)}</entity_type_name>
<id>${Albums.replaceSpecialChars(labels[j].id)}</id>
<resource_url>${Albums.replaceSpecialChars(
labels[j].resource_url
)}</resource_url>
<thumbnail_url>${Albums.replaceSpecialChars(
labels[j].thumbnail_url
)}</thumbnail_url>
</label>
`;
}
for (let j = 0; j < series.length; j += 1) {
serieList += `<serie>
<name>${Albums.replaceSpecialChars(series[j].name)}</name>
<catno>${Albums.replaceSpecialChars(series[j].catno)}</catno>
<entity_type>${Albums.replaceSpecialChars(
series[j].entity_type
)}</entity_type>
<entity_type_name>${Albums.replaceSpecialChars(
series[j].entity_type_name
)}</entity_type_name>
<id>${Albums.replaceSpecialChars(series[j].id)}</id>
<resource_url>${Albums.replaceSpecialChars(
series[j].resource_url
)}</resource_url>
<thumbnail_url>${Albums.replaceSpecialChars(
series[j].thumbnail_url
)}</thumbnail_url>
</serie>
`;
}
for (let j = 0; j < companies.length; j += 1) {
companiesList += `<company>
<name>${Albums.replaceSpecialChars(companies[j].name)}</name>
<catno>${Albums.replaceSpecialChars(companies[j].catno)}</catno>
<entity_type>${Albums.replaceSpecialChars(
companies[j].entity_type
)}</entity_type>
<entity_type_name>${Albums.replaceSpecialChars(
companies[j].entity_type_name
)}</entity_type_name>
<id>${Albums.replaceSpecialChars(companies[j].id)}</id>
<resource_url>${Albums.replaceSpecialChars(
companies[j].resource_url
)}</resource_url>
<thumbnail_url>${Albums.replaceSpecialChars(
companies[j].thumbnail_url
)}</thumbnail_url>
</company>
`;
}
for (let j = 0; j < formats.length; j += 1) {
let descriptions = "";
if (formats[j].descriptions) {
for (
let k = 0;
k < formats[j].descriptions.length;
k += 1
) {
descriptions += `<description>${formats[j].descriptions[k]}</description>
`;
}
}
formatsList += `<format>
<name>${Albums.replaceSpecialChars(formats[j].name)}</name>
<qte>${Albums.replaceSpecialChars(formats[j].qty)}</qte>
<text>${Albums.replaceSpecialChars(formats[j].text)}</text>
<descriptions>
${descriptions}
</descriptions>
</format>
`;
}
for (let j = 0; j < identifiers.length; j += 1) {
identifiersList += `<identifier>
<type>${Albums.replaceSpecialChars(identifiers[j].type)}</type>
<value>${Albums.replaceSpecialChars(
identifiers[j].value
)}</value>
<description>${Albums.replaceSpecialChars(
identifiers[j].description
)}</description>
</identifier>
`;
}
for (let j = 0; j < videos.length; j += 1) {
videosList += `<video embed="${videos[j].embed}">
<uri>${Albums.replaceSpecialChars(videos[j].uri)}</uri>
<title>${Albums.replaceSpecialChars(videos[j].title)}</title>
<description>${Albums.replaceSpecialChars(
videos[j].description
)}</description>
<duration>${Albums.replaceSpecialChars(
videos[j].duration
)}</duration>
</video>
`;
}
for (let j = 0; j < genres.length; j += 1) {
genresList += `<genre>${Albums.replaceSpecialChars(
genres[j]
)}</genre>
`;
}
for (let j = 0; j < styles.length; j += 1) {
stylesList += `<style>${Albums.replaceSpecialChars(
styles[j]
)}</style>
`;
}
for (let j = 0; j < tracklist.length; j += 1) {
tracklistList += `<tracklist position="${
tracklist[j].position
}" type="${tracklist[j].type_}" duration="${
tracklist[j].duration
}">
${Albums.replaceSpecialChars(tracklist[j].title)}
</tracklist>
`;
}
for (let j = 0; j < extraartists.length; j += 1) {
extraartistsList += `<extraartist>
<name>${Albums.replaceSpecialChars(extraartists[j].name)}</name>
<anv>${Albums.replaceSpecialChars(extraartists[j].anv)}</anv>
<join>${Albums.replaceSpecialChars(extraartists[j].join)}</join>
<role>${Albums.replaceSpecialChars(extraartists[j].role)}</role>
<tracks>${Albums.replaceSpecialChars(
extraartists[j].tracks
)}</tracks>
<id>${Albums.replaceSpecialChars(extraartists[j].id)}</id>
<resource_url>${Albums.replaceSpecialChars(
extraartists[j].resource_url
)}</resource_url>
<thumbnail_url>${Albums.replaceSpecialChars(
extraartists[j].thumbnail_url
)}</thumbnail_url>
</extraartist>
`;
}
for (let j = 0; j < images.length; j += 1) {
imagesList += `<image type="${images[j].type}" width="${
images[j].width
}" height="${images[j].height}">
<uri>${Albums.replaceSpecialChars(images[j].uri)}</uri>
<resource_url>${Albums.replaceSpecialChars(
images[j].resource_url
)}</resource_url>
<uri150>${Albums.replaceSpecialChars(
images[j].resource_url
)}</uri150>
</image>
`;
}
data += `
<album>
<discogId>${discogsId}</discogId>
<title>${Albums.replaceSpecialChars(title)}</title>
<artists_sort>${Albums.replaceSpecialChars(artists_sort)}</artists_sort>
<artists>
${artistsList}
</artists>
<year>${year}</year>
<country>${Albums.replaceSpecialChars(country)}</country>
<released>${released}</released>
<uri>${uri}</uri>
<thumb>${thumb}</thumb>
<labels>
${labelList}
</labels>
<series>
${serieList}
</series>
<companies>
${companiesList}
</companies>
<formats>
${formatsList}
</formats>
<notes>${Albums.replaceSpecialChars(notes)}</notes>
<identifiers>
${identifiersList}
</identifiers>
<videos>
${videosList}
</videos>
<genres>
${genresList}
</genres>
<styles>
${stylesList}
</styles>
<tracklist>
${tracklistList}
</tracklist>
<extraartists>
${extraartistsList}
</extraartists>
<images>
${imagesList}
</images>
</album>`;
}
return `${data}</albums>`;
}
/**
* Méthode permettant de convertir les rows en csv pour importer dans MusicTopus
* @param {Array} rows
*
* @return {string}
*/
static async convertToMusicTopus(rows) {
let data = "itemId;createdAt;updatedAt\n\r";
for (let i = 0; i < rows.length; i += 1) {
const { discogsId, createdAt, updatedAt } = rows[i];
data += `${discogsId};${createdAt};${updatedAt}\n\r`;
}
data += "v1.0";
return data;
}
/**
* Méthode permettant d'ajouter un album dans une collection
* @param {Object} req
@ -482,7 +32,18 @@ class Albums extends Pages {
const album = new AlbumsModel(data);
return album.save();
await album.save();
const jobData = {
model: "Albums",
id: album._id,
};
const job = new JobsModel(jobData);
job.save();
return album;
}
/**
@ -593,13 +154,13 @@ class Albums extends Pages {
switch (exportFormat) {
case "csv":
return Albums.convertToCsv(rows);
return Export.convertToCsv(rows);
case "xls":
return Albums.convertToXls(rows);
return Export.convertToXls(rows);
case "xml":
return Albums.convertToXml(rows);
return Export.convertToXml(rows);
case "musictopus":
return Albums.convertToMusicTopus(rows);
return Export.convertToMusicTopus(rows);
case "json":
default:
return {

453
src/middleware/Export.js Normal file
View file

@ -0,0 +1,453 @@
import momenttz from "moment-timezone";
import xl from "excel4node";
class Export {
/**
* Méthode permettant de remplacer certains cartactères par leur équivalents html
* @param {String} str
*
* @return {String}
*/
static replaceSpecialChars(str) {
if (!str) {
return "";
}
let final = str.toString();
const find = ["&", "<", ">"];
const replace = ["&amp;", "&lt;", "&gt;"];
for (let i = 0; i < find.length; i += 1) {
final = final.replace(new RegExp(find[i], "g"), replace[i]);
}
return final;
}
/**
* Méthode permettant de convertir les rows en csv
* @param {Array} rows
*
* @return {string}
*/
static async convertToCsv(rows) {
let data =
"Artiste;Titre;Genre;Styles;Pays;Année;Date de sortie;Format\n\r";
for (let i = 0; i < rows.length; i += 1) {
const {
artists_sort,
title,
genres,
styles,
country,
year,
released,
formats,
} = rows[i];
let format = "";
for (let j = 0; j < formats.length; j += 1) {
format += `${format !== "" ? ", " : ""}${formats[j].name}`;
}
data += `${artists_sort};${title};${genres.join()};${styles.join()};${country};${year};${released};${format}\n\r`;
}
return data;
}
/**
* Méthode permettant de convertir les rows en fichier xls
* @param {Array} rows
*
* @return {Object}
*/
static async convertToXls(rows) {
const wb = new xl.Workbook();
const ws = wb.addWorksheet("MusicTopus");
const headerStyle = wb.createStyle({
font: {
color: "#FFFFFF",
size: 11,
},
fill: {
type: "pattern",
patternType: "solid",
bgColor: "#595959",
fgColor: "#595959",
},
});
const style = wb.createStyle({
font: {
color: "#000000",
size: 11,
},
numberFormat: "0000",
});
const header = [
"Artiste",
"Titre",
"Genre",
"Styles",
"Pays",
"Année",
"Date de sortie",
"Format",
];
for (let i = 0; i < header.length; i += 1) {
ws.cell(1, i + 1)
.string(header[i])
.style(headerStyle);
}
for (let i = 0; i < rows.length; i += 1) {
const currentRow = i + 2;
const {
artists_sort,
title,
genres,
styles,
country,
year,
released,
formats,
} = rows[i];
let format = "";
for (let j = 0; j < formats.length; j += 1) {
format += `${format !== "" ? ", " : ""}${formats[j].name}`;
}
ws.cell(currentRow, 1).string(artists_sort).style(style);
ws.cell(currentRow, 2).string(title).style(style);
ws.cell(currentRow, 3).string(genres.join()).style(style);
ws.cell(currentRow, 4).string(styles.join()).style(style);
if (country) {
ws.cell(currentRow, 5).string(country).style(style);
}
if (year) {
ws.cell(currentRow, 6).number(year).style(style);
}
if (released) {
ws.cell(currentRow, 7)
.date(momenttz.tz(released, "Europe/Paris").hour(12))
.style({ numberFormat: "dd/mm/yyyy" });
}
ws.cell(currentRow, 8).string(format).style(style);
}
return wb;
}
/**
* Méthode permettant de convertir les rows en csv pour importer dans MusicTopus
* @param {Array} rows
*
* @return {string}
*/
static async convertToXml(rows) {
let data = '<?xml version="1.0" encoding="UTF-8"?>\n\r<albums>';
for (let i = 0; i < rows.length; i += 1) {
const {
discogsId,
year,
released,
uri,
artists,
artists_sort,
labels,
series,
companies,
formats,
title,
country,
notes,
identifiers,
videos,
genres,
styles,
tracklist,
extraartists,
images,
thumb,
} = rows[i];
let artistsList = "";
let labelList = "";
let serieList = "";
let companiesList = "";
let formatsList = "";
let identifiersList = "";
let videosList = "";
let genresList = "";
let stylesList = "";
let tracklistList = "";
let extraartistsList = "";
let imagesList = "";
for (let j = 0; j < artists.length; j += 1) {
artistsList += `<artist>
<name>${Export.replaceSpecialChars(artists[j].name)}</name>
<anv>${Export.replaceSpecialChars(artists[j].anv)}</anv>
<join>${Export.replaceSpecialChars(artists[j].join)}</join>
<role>${Export.replaceSpecialChars(artists[j].role)}</role>
<tracks>${Export.replaceSpecialChars(artists[j].tracks)}</tracks>
<id>${Export.replaceSpecialChars(artists[j].id)}</id>
<resource_url>${Export.replaceSpecialChars(
artists[j].resource_url
)}</resource_url>
<thumbnail_url>${Export.replaceSpecialChars(
artists[j].thumbnail_url
)}</thumbnail_url>
</artist>`;
}
for (let j = 0; j < labels.length; j += 1) {
labelList += `<label>
<name>${Export.replaceSpecialChars(labels[j].name)}</name>
<catno>${Export.replaceSpecialChars(labels[j].catno)}</catno>
<entity_type>${Export.replaceSpecialChars(
labels[j].entity_type
)}</entity_type>
<entity_type_name>${Export.replaceSpecialChars(
labels[j].entity_type
)}</entity_type_name>
<id>${Export.replaceSpecialChars(labels[j].id)}</id>
<resource_url>${Export.replaceSpecialChars(
labels[j].resource_url
)}</resource_url>
<thumbnail_url>${Export.replaceSpecialChars(
labels[j].thumbnail_url
)}</thumbnail_url>
</label>
`;
}
for (let j = 0; j < series.length; j += 1) {
serieList += `<serie>
<name>${Export.replaceSpecialChars(series[j].name)}</name>
<catno>${Export.replaceSpecialChars(series[j].catno)}</catno>
<entity_type>${Export.replaceSpecialChars(
series[j].entity_type
)}</entity_type>
<entity_type_name>${Export.replaceSpecialChars(
series[j].entity_type_name
)}</entity_type_name>
<id>${Export.replaceSpecialChars(series[j].id)}</id>
<resource_url>${Export.replaceSpecialChars(
series[j].resource_url
)}</resource_url>
<thumbnail_url>${Export.replaceSpecialChars(
series[j].thumbnail_url
)}</thumbnail_url>
</serie>
`;
}
for (let j = 0; j < companies.length; j += 1) {
companiesList += `<company>
<name>${Export.replaceSpecialChars(companies[j].name)}</name>
<catno>${Export.replaceSpecialChars(companies[j].catno)}</catno>
<entity_type>${Export.replaceSpecialChars(
companies[j].entity_type
)}</entity_type>
<entity_type_name>${Export.replaceSpecialChars(
companies[j].entity_type_name
)}</entity_type_name>
<id>${Export.replaceSpecialChars(companies[j].id)}</id>
<resource_url>${Export.replaceSpecialChars(
companies[j].resource_url
)}</resource_url>
<thumbnail_url>${Export.replaceSpecialChars(
companies[j].thumbnail_url
)}</thumbnail_url>
</company>
`;
}
for (let j = 0; j < formats.length; j += 1) {
let descriptions = "";
if (formats[j].descriptions) {
for (
let k = 0;
k < formats[j].descriptions.length;
k += 1
) {
descriptions += `<description>${formats[j].descriptions[k]}</description>
`;
}
}
formatsList += `<format>
<name>${Export.replaceSpecialChars(formats[j].name)}</name>
<qte>${Export.replaceSpecialChars(formats[j].qty)}</qte>
<text>${Export.replaceSpecialChars(formats[j].text)}</text>
<descriptions>
${descriptions}
</descriptions>
</format>
`;
}
for (let j = 0; j < identifiers.length; j += 1) {
identifiersList += `<identifier>
<type>${Export.replaceSpecialChars(identifiers[j].type)}</type>
<value>${Export.replaceSpecialChars(identifiers[j].value)}</value>
<description>${Export.replaceSpecialChars(
identifiers[j].description
)}</description>
</identifier>
`;
}
for (let j = 0; j < videos.length; j += 1) {
videosList += `<video embed="${videos[j].embed}">
<uri>${Export.replaceSpecialChars(videos[j].uri)}</uri>
<title>${Export.replaceSpecialChars(videos[j].title)}</title>
<description>${Export.replaceSpecialChars(
videos[j].description
)}</description>
<duration>${Export.replaceSpecialChars(
videos[j].duration
)}</duration>
</video>
`;
}
for (let j = 0; j < genres.length; j += 1) {
genresList += `<genre>${Export.replaceSpecialChars(
genres[j]
)}</genre>
`;
}
for (let j = 0; j < styles.length; j += 1) {
stylesList += `<style>${Export.replaceSpecialChars(
styles[j]
)}</style>
`;
}
for (let j = 0; j < tracklist.length; j += 1) {
tracklistList += `<tracklist position="${
tracklist[j].position
}" type="${tracklist[j].type_}" duration="${
tracklist[j].duration
}">
${Export.replaceSpecialChars(tracklist[j].title)}
</tracklist>
`;
}
for (let j = 0; j < extraartists.length; j += 1) {
extraartistsList += `<extraartist>
<name>${Export.replaceSpecialChars(extraartists[j].name)}</name>
<anv>${Export.replaceSpecialChars(extraartists[j].anv)}</anv>
<join>${Export.replaceSpecialChars(extraartists[j].join)}</join>
<role>${Export.replaceSpecialChars(extraartists[j].role)}</role>
<tracks>${Export.replaceSpecialChars(
extraartists[j].tracks
)}</tracks>
<id>${Export.replaceSpecialChars(extraartists[j].id)}</id>
<resource_url>${Export.replaceSpecialChars(
extraartists[j].resource_url
)}</resource_url>
<thumbnail_url>${Export.replaceSpecialChars(
extraartists[j].thumbnail_url
)}</thumbnail_url>
</extraartist>
`;
}
for (let j = 0; j < images.length; j += 1) {
imagesList += `<image type="${images[j].type}" width="${
images[j].width
}" height="${images[j].height}">
<uri>${Export.replaceSpecialChars(images[j].uri)}</uri>
<resource_url>${Export.replaceSpecialChars(
images[j].resource_url
)}</resource_url>
<uri150>${Export.replaceSpecialChars(
images[j].resource_url
)}</uri150>
</image>
`;
}
data += `
<album>
<discogId>${discogsId}</discogId>
<title>${Export.replaceSpecialChars(title)}</title>
<artists_sort>${Export.replaceSpecialChars(artists_sort)}</artists_sort>
<artists>
${artistsList}
</artists>
<year>${year}</year>
<country>${Export.replaceSpecialChars(country)}</country>
<released>${released}</released>
<uri>${uri}</uri>
<thumb>${thumb}</thumb>
<labels>
${labelList}
</labels>
<series>
${serieList}
</series>
<companies>
${companiesList}
</companies>
<formats>
${formatsList}
</formats>
<notes>${Export.replaceSpecialChars(notes)}</notes>
<identifiers>
${identifiersList}
</identifiers>
<videos>
${videosList}
</videos>
<genres>
${genresList}
</genres>
<styles>
${stylesList}
</styles>
<tracklist>
${tracklistList}
</tracklist>
<extraartists>
${extraartistsList}
</extraartists>
<images>
${imagesList}
</images>
</album>`;
}
return `${data}</albums>`;
}
/**
* Méthode permettant de convertir les rows en csv pour importer dans MusicTopus
* @param {Array} rows
*
* @return {string}
*/
static async convertToMusicTopus(rows) {
let data = "itemId;createdAt;updatedAt\n\r";
for (let i = 0; i < rows.length; i += 1) {
const { discogsId, createdAt, updatedAt } = rows[i];
data += `${discogsId};${createdAt};${updatedAt}\n\r`;
}
data += "v1.0";
return data;
}
}
export default Export;

128
src/middleware/Jobs.js Normal file
View file

@ -0,0 +1,128 @@
/* eslint-disable no-await-in-loop */
import ErrorEvent from "../libs/error";
import { uploadFromUrl } from "../libs/aws";
import { getAlbumDetails } from "../helpers";
import JobsModel from "../models/jobs";
import AlbumsModel from "../models/albums";
class Jobs {
/**
* Méthode permettant de télécharger toute les images d'un album
* @param {ObjectId} itemId
*/
static async importAlbumAssets(itemId) {
const album = await AlbumsModel.findById(itemId);
if (!album) {
throw new ErrorEvent(
404,
"Item non trouvé",
`L'album avant l'id ${itemId} n'existe plus dans la collection`
);
}
const item = await getAlbumDetails(album.discogsId);
if (!item) {
throw new ErrorEvent(
404,
"Erreur de communication",
"Erreur lors de la récupération des informations sur Discogs"
);
}
if (item.thumb) {
album.thumb = await uploadFromUrl(item.thumb);
album.thumbType = "local";
}
const { images } = item;
if (images && images.length > 0) {
for (let i = 0; i < images.length; i += 1) {
images[i].uri150 = await uploadFromUrl(images[i].uri150);
images[i].uri = await uploadFromUrl(images[i].uri);
}
}
album.images = images;
await album.save();
return true;
}
/**
* Point d'entrée
* @param {String} state
*
* @return {Object}
*/
async run(state = "NEW") {
const job = await JobsModel.findOne({
state,
tries: {
$lte: 5,
},
});
if (!job) {
return { message: "All jobs done" };
}
job.state = "IN-PROGRESS";
await job.save();
try {
switch (job.model) {
case "Albums":
await Jobs.importAlbumAssets(job.id);
break;
default:
throw new ErrorEvent(
500,
"Job inconnu",
`Le job avec l'id ${job._id} n'est pas un job valide`
);
}
job.state = "SUCCESS";
await job.save();
return this.run(state);
} catch (err) {
job.state = "ERROR";
job.lastTry = new Date();
job.lastErrorMessage = err.message;
job.tries += 1;
await job.save();
throw err;
}
}
/**
* Méthode permettant de créer tous les jobs
*
* @return {Object}
*/
static async populate() {
const albums = await AlbumsModel.find();
for (let i = 0; i < albums.length; i += 1) {
const jobData = {
model: "Albums",
id: albums[i]._id,
};
const job = new JobsModel(jobData);
await job.save();
}
return { message: `${albums.length} jobs ajouté à la file d'attente` };
}
}
export default Jobs;

View file

@ -29,6 +29,7 @@ const AlbumSchema = new mongoose.Schema(
extraartists: Array,
images: Array,
thumb: String,
thumbType: String,
},
{ timestamps: true }
);

24
src/models/jobs.js Normal file
View file

@ -0,0 +1,24 @@
import mongoose from "mongoose";
const { Schema } = mongoose;
const JobSchema = new mongoose.Schema(
{
model: String,
id: Schema.Types.ObjectId,
state: {
type: String,
enum: ["NEW", "IN-PROGRESS", "ERROR", "SUCCESS"],
default: "NEW",
},
lastTry: Date,
lastErrorMessage: String,
tries: {
type: Number,
default: 0,
},
},
{ timestamps: true }
);
export default mongoose.model("Jobs", JobSchema);

40
src/routes/jobs.js Normal file
View file

@ -0,0 +1,40 @@
import express from "express";
import passport from "passport";
import Jobs from "../middleware/Jobs";
// eslint-disable-next-line new-cap
const router = express.Router();
router.route("/").get(
passport.authenticate(["jobs"], {
session: false,
}),
async (req, res, next) => {
try {
const job = new Jobs();
const data = await job.run(req.query.state);
return res.status(200).json(data).end();
} catch (err) {
return next(err);
}
}
);
router.route("/populate").get(
passport.authenticate(["jobs"], {
session: false,
}),
async (req, res, next) => {
try {
const data = await Jobs.populate();
return res.status(200).json(data).end();
} catch (err) {
return next(err);
}
}
);
export default router;