From 3bbb23dcebea3122face8e342ce2c608f3faf23f Mon Sep 17 00:00:00 2001 From: Stepan Usatiuk Date: Fri, 16 Oct 2020 16:54:41 +0000 Subject: [PATCH] upload photos in parallel (not all at once) --- frontend/src/AppToaster.tsx | 6 +- frontend/src/redux/photos/actions.ts | 62 ++++++++++++-- frontend/src/redux/photos/reducer.ts | 112 ++++++++++++++++++++++--- frontend/src/redux/photos/sagas.ts | 118 ++++++++++++++++++--------- 4 files changed, 239 insertions(+), 59 deletions(-) diff --git a/frontend/src/AppToaster.tsx b/frontend/src/AppToaster.tsx index 0a4df19..12a7412 100644 --- a/frontend/src/AppToaster.tsx +++ b/frontend/src/AppToaster.tsx @@ -1,4 +1,5 @@ import { Position, Toaster } from "@blueprintjs/core"; +import { isNumber } from "class-validator"; import { IPhotoReqJSON } from "~../../src/entity/Photo"; export const AppToaster = Toaster.create({ @@ -43,11 +44,12 @@ export function showPhotoCreateFailToast(f: File, e: string): void { } export function showPhotoUploadJSONFailToast( - p: IPhotoReqJSON, + p: IPhotoReqJSON | number, e: string, ): void { + const photoMsg = typeof p === "number" ? p : p.hash; AppToaster.show({ - message: `Failed to upload ${p.hash}: ${e}`, + message: `Failed to upload ${photoMsg}: ${e}`, intent: "danger", timeout: 1000, }); diff --git a/frontend/src/redux/photos/actions.ts b/frontend/src/redux/photos/actions.ts index 94ec97c..050dd0d 100644 --- a/frontend/src/redux/photos/actions.ts +++ b/frontend/src/redux/photos/actions.ts @@ -14,8 +14,12 @@ export enum PhotoTypes { PHOTO_LOAD_SUCCESS = "PHOTO_LOAD_SUCCESS", PHOTO_LOAD_FAIL = "PHOTO_LOAD_FAIL", PHOTOS_UPLOAD_START = "PHOTOS_UPLOAD", + PHOTO_CREATE_QUEUE = "PHOTO_CREATE_QUEUE", + PHOTO_CREATE_START = "PHOTO_CREATE_START", PHOTO_CREATE_SUCCESS = "PHOTO_CREATE_SUCCESS", PHOTO_CREATE_FAIL = "PHOTO_CREATE_FAIL", + PHOTO_UPLOAD_QUEUE = "PHOTO_UPLOAD_QUEUE", + PHOTO_UPLOAD_START = "PHOTO_UPLOAD_START", PHOTO_UPLOAD_SUCCESS = "PHOTO_UPLOAD_SUCCESS", PHOTO_UPLOAD_FAIL = "PHOTO_UPLOAD_FAIL", PHOTOS_START_FETCHING_SPINNER = "PHOTOS_START_FETCHING_SPINNER", @@ -60,6 +64,28 @@ export interface IPhotosUploadStartAction extends Action { files: FileList; } +export interface IPhotoCreateQueue extends Action { + type: PhotoTypes.PHOTO_CREATE_QUEUE; + file: File; +} + +export interface IPhotoUploadQueue extends Action { + type: PhotoTypes.PHOTO_UPLOAD_QUEUE; + file: File; + id: number; +} + +export interface IPhotoCreateStart extends Action { + type: PhotoTypes.PHOTO_CREATE_START; + file: File; +} + +export interface IPhotoUploadStart extends Action { + type: PhotoTypes.PHOTO_UPLOAD_START; + file: File; + id: number; +} + export interface IPhotoUploadSuccessAction extends Action { type: PhotoTypes.PHOTO_UPLOAD_SUCCESS; photo: IPhotoReqJSON; @@ -67,13 +93,14 @@ export interface IPhotoUploadSuccessAction extends Action { export interface IPhotoUploadFailAction extends Action { type: PhotoTypes.PHOTO_UPLOAD_FAIL; - photo: IPhotoReqJSON; + photo: IPhotoReqJSON | number; error: string; } export interface IPhotoCreateSuccessAction extends Action { type: PhotoTypes.PHOTO_CREATE_SUCCESS; photo: IPhotoReqJSON; + file: File; } export interface IPhotoCreateFailAction extends Action { @@ -107,6 +134,22 @@ export interface IPhotosStartFetchingSpinner extends Action { type: PhotoTypes.PHOTOS_START_FETCHING_SPINNER; } +export function photoCreateQueue(file: File): IPhotoCreateQueue { + return { type: PhotoTypes.PHOTO_CREATE_QUEUE, file }; +} + +export function photoUploadQueue(file: File, id: number): IPhotoUploadQueue { + return { type: PhotoTypes.PHOTO_UPLOAD_QUEUE, file, id }; +} + +export function photoCreateStart(file: File): IPhotoCreateStart { + return { type: PhotoTypes.PHOTO_CREATE_START, file }; +} + +export function photoUploadStart(file: File, id: number): IPhotoUploadStart { + return { type: PhotoTypes.PHOTO_UPLOAD_START, file, id }; +} + export function photosLoadStart(): IPhotosLoadStartAction { return { type: PhotoTypes.PHOTOS_LOAD_START }; } @@ -126,7 +169,7 @@ export function photoUploadSuccess( } export function photoUploadFail( - photo: IPhotoReqJSON, + photo: IPhotoReqJSON | number, error: string, ): IPhotoUploadFailAction { showPhotoUploadJSONFailToast(photo, error); @@ -134,7 +177,7 @@ export function photoUploadFail( } export function photoUploadFailWithFile( - photo: IPhotoReqJSON, + photo: IPhotoReqJSON | number, file: File, error: string, ): IPhotoUploadFailAction { @@ -144,8 +187,9 @@ export function photoUploadFailWithFile( export function photoCreateSuccess( photo: IPhotoReqJSON, + file: File, ): IPhotoCreateSuccessAction { - return { type: PhotoTypes.PHOTO_CREATE_SUCCESS, photo }; + return { type: PhotoTypes.PHOTO_CREATE_SUCCESS, photo, file }; } export function photoCreateFail( @@ -172,8 +216,8 @@ export function photoLoadSuccess( return { type: PhotoTypes.PHOTO_LOAD_SUCCESS, photo }; } -export function photoLoadFail(id:number,error: string): IPhotoLoadFailAction { - return { type: PhotoTypes.PHOTO_LOAD_FAIL,id, error }; +export function photoLoadFail(id: number, error: string): IPhotoLoadFailAction { + return { type: PhotoTypes.PHOTO_LOAD_FAIL, id, error }; } export function photoDeleteStart( @@ -219,4 +263,8 @@ export type PhotoAction = | IPhotoDeleteCancelAction | IPhotoLoadFailAction | IPhotoLoadStartAction - | IPhotoLoadSuccessAction; + | IPhotoLoadSuccessAction + | IPhotoUploadQueue + | IPhotoCreateQueue + | IPhotoCreateStart + | IPhotoUploadStart; diff --git a/frontend/src/redux/photos/reducer.ts b/frontend/src/redux/photos/reducer.ts index 1118d2c..4bad575 100644 --- a/frontend/src/redux/photos/reducer.ts +++ b/frontend/src/redux/photos/reducer.ts @@ -18,6 +18,11 @@ export interface IPhotosState { overviewFetchingError: string | null; overviewFetchingSpinner: boolean; + photoCreateQueue: File[]; + photosCreating: number; + photoUploadQueue: Record; + photosUploading: number; + deleteCache: Record; } @@ -28,6 +33,11 @@ const defaultPhotosState: IPhotosState = { overviewFetchingError: null, overviewFetchingSpinner: false, + photoCreateQueue: [], + photosCreating: 0, + photoUploadQueue: {}, + photosUploading: 0, + photoStates: {}, deleteCache: {}, @@ -104,31 +114,102 @@ export const photosReducer: Reducer = ( photoStates, }; } - case PhotoTypes.PHOTO_CREATE_SUCCESS: + case PhotoTypes.PHOTO_CREATE_QUEUE: { + const { photoCreateQueue } = state; + return { + ...state, + photoCreateQueue: [...photoCreateQueue, action.file], + }; + break; + } + case PhotoTypes.PHOTO_CREATE_START: { + const { photoCreateQueue } = state; + const cleanQueue = photoCreateQueue.filter((f) => f != action.file); + + return { + ...state, + photosCreating: state.photosCreating + 1, + photoCreateQueue: cleanQueue, + }; + break; + } + case PhotoTypes.PHOTO_UPLOAD_START: { + const newQueue = state.photoUploadQueue; + delete newQueue[action.id]; + + return { + ...state, + photosUploading: state.photosUploading + 1, + photoUploadQueue: newQueue, + }; + break; + } + case PhotoTypes.PHOTO_UPLOAD_QUEUE: { + const newQueue = state.photoUploadQueue; + newQueue[action.id] = action.file; + return { + ...state, + photoUploadQueue: newQueue, + }; + break; + } + case PhotoTypes.PHOTO_CREATE_SUCCESS: { + const { photoCreateQueue } = state; + const cleanQueue = photoCreateQueue.filter((f) => f != action.file); + if (state.photos) { const photos = state.photos; const photosNoDup = photos.filter( (p) => p.id !== action.photo.id, ); const updPhotos = [action.photo, ...photosNoDup]; - return { ...state, photos: updPhotos }; + return { + ...state, + photos: updPhotos, + photoCreateQueue: cleanQueue, + photosCreating: state.photosCreating - 1, + }; } else { - return state; + return { + ...state, + photoCreateQueue: cleanQueue, + photosCreating: state.photosCreating - 1, + }; } - case PhotoTypes.PHOTO_CREATE_FAIL: + } + case PhotoTypes.PHOTO_CREATE_FAIL: { // TODO: Handle photo create fail - return state; - case PhotoTypes.PHOTO_UPLOAD_SUCCESS: + const { photoCreateQueue } = state; + const cleanQueue = photoCreateQueue.filter((f) => f != action.file); + return { + ...state, + photoCreateQueue: cleanQueue, + photosCreating: state.photosCreating - 1, + }; + } + case PhotoTypes.PHOTO_UPLOAD_SUCCESS: { + const newQueue = state.photoUploadQueue; + delete newQueue[action.photo.id]; if (state.photos) { const photos = state.photos; const photosNoDup = photos.filter( (p) => p.id !== action.photo.id, ); const updPhotos = [action.photo, ...photosNoDup]; - return { ...state, photos: updPhotos }; + return { + ...state, + photos: updPhotos, + photoUploadQueue: newQueue, + photosUploading: state.photosUploading - 1, + }; } else { - return state; + return { + ...state, + photoUploadQueue: newQueue, + photosUploading: state.photosUploading - 1, + }; } + } case PhotoTypes.PHOTO_DELETE_START: if (state.photos) { const photos = state.photos; @@ -172,9 +253,20 @@ export const photosReducer: Reducer = ( return { ...state, deleteCache: delCache, photos }; break; } - case PhotoTypes.PHOTO_UPLOAD_FAIL: + case PhotoTypes.PHOTO_UPLOAD_FAIL: { // TODO: Handle photo upload fail - return state; + const newQueue = state.photoUploadQueue; + if (typeof action.photo === "number") { + delete newQueue[action.photo]; + } else { + delete newQueue[action.photo.id]; + } + return { + ...state, + photoUploadQueue: newQueue, + photosUploading: state.photosUploading - 1, + }; + } default: return state; } diff --git a/frontend/src/redux/photos/sagas.ts b/frontend/src/redux/photos/sagas.ts index 6fccb55..4e45350 100644 --- a/frontend/src/redux/photos/sagas.ts +++ b/frontend/src/redux/photos/sagas.ts @@ -9,6 +9,7 @@ import { takeLatest, takeEvery, take, + select, } from "redux-saga/effects"; import * as SparkMD5 from "spark-md5"; import { @@ -23,6 +24,8 @@ import { IPhotoLoadStartAction, IPhotosUploadStartAction, photoCreateFail, + photoCreateQueue, + photoCreateStart, photoCreateSuccess, photoDeleteFail, photoDeleteSuccess, @@ -34,6 +37,8 @@ import { PhotoTypes, photoUploadFail, photoUploadFailWithFile, + photoUploadQueue, + photoUploadStart, photoUploadSuccess, } from "./actions"; import { IPhotosNewRespBody } from "~../../src/routes/photos"; @@ -160,57 +165,84 @@ function* photoLoad(action: IPhotoLoadStartAction) { } } -function* photoUpload(f: File) { - try { - const hash = yield call(computeChecksumMd5, f); - const size = yield call(computeSize, f); - const format = f.type; - - const { response, timeout } = yield race({ - response: call(createPhoto, hash, size, format), - timeout: delay(10000), - }); - - if (timeout) { - yield put(photoCreateFail(f, "Timeout")); +function* photoCreate() { + const store = yield select(); + const photosCreating = store.photos.photosCreating; + if (photosCreating < 2) { + const createQueue = store.photos.photoCreateQueue as File[]; + if (createQueue.length === 0) { return; } - if (response.data || response.error === "Photo already exists") { - const photo = (response as IPhotosNewRespBody).data; - yield put(photoCreateSuccess(photo)); + const f = createQueue[0]; + yield put(photoCreateStart(f)); + try { + const hash = yield call(computeChecksumMd5, f); + const size = yield call(computeSize, f); + const format = f.type; - try { - const { response, timeout } = yield race({ - response: call(uploadPhoto, f, photo.id), - timeout: delay(10000), - }); + const { response, timeout } = yield race({ + response: call(createPhoto, hash, size, format), + timeout: delay(10000), + }); - if (timeout) { - yield put(photoUploadFailWithFile(photo, f, "Timeout")); - return; - } - if (response.data) { - const photo = response.data; - yield put(photoUploadSuccess(photo)); - } else { - yield put( - photoUploadFailWithFile(photo, f, response.error), - ); - } - } catch (e) { - yield put(photoUploadFailWithFile(photo, f, "Internal error")); + if (timeout) { + yield put(photoCreateFail(f, "Timeout")); + return; } - } else { - yield put(photoCreateFail(f, response.error)); + if (response.data || response.error === "Photo already exists") { + const photo = (response as IPhotosNewRespBody).data; + yield put(photoCreateSuccess(photo, f)); + yield put(photoUploadQueue(f, photo.id)); + } else { + yield put(photoCreateFail(f, response.error)); + } + } catch (e) { + yield put(photoCreateFail(f, "Internal error")); + } + } +} + +function* photoUpload() { + const store = yield select(); + const photosUploading = store.photos.photosUploading; + if (photosUploading < 2) { + const createQueue = store.photos.photoUploadQueue as Record< + number, + File + >; + if (Object.keys(createQueue).length === 0) { + return; + } + const pId = parseInt(Object.keys(createQueue)[0]); + const f = createQueue[pId]; + yield put(photoUploadStart(f, pId)); + try { + const { response, timeout } = yield race({ + response: call(uploadPhoto, f, pId), + timeout: delay(10000), + }); + + if (timeout) { + yield put(photoUploadFailWithFile(pId, f, "Timeout")); + return; + } + if (response.data) { + const photo = response.data; + yield put(photoUploadSuccess(photo)); + } else { + yield put(photoUploadFailWithFile(pId, f, response.error)); + } + } catch (e) { + yield put(photoUploadFailWithFile(pId, f, "Internal error")); } - } catch (e) { - yield put(photoCreateFail(f, "Internal error")); } } function* photosUpload(action: IPhotosUploadStartAction) { const files = Array.from(action.files); - yield all(files.map((f) => call(photoUpload, f))); + for (const file of files) { + yield put(photoCreateQueue(file)); + } } function* photoDelete(action: IPhotoDeleteStartAction) { @@ -250,5 +282,11 @@ export function* photosSaga() { takeLatest(PhotoTypes.PHOTOS_UPLOAD_START, photosUpload), takeLatest(PhotoTypes.PHOTO_LOAD_START, photoLoad), takeEvery(PhotoTypes.PHOTO_DELETE_START, photoDelete), + takeEvery(PhotoTypes.PHOTO_CREATE_QUEUE, photoCreate), + takeEvery(PhotoTypes.PHOTO_CREATE_SUCCESS, photoCreate), + takeEvery(PhotoTypes.PHOTO_CREATE_FAIL, photoCreate), + takeEvery(PhotoTypes.PHOTO_UPLOAD_QUEUE, photoUpload), + takeEvery(PhotoTypes.PHOTO_UPLOAD_SUCCESS, photoUpload), + takeEvery(PhotoTypes.PHOTO_UPLOAD_FAIL, photoUpload), ]); }