upload photos in parallel (not all at once)

frontend/src/AppToaster.tsx

@@ -1,4 +1,5 @@
 import { Position, Toaster } from "@blueprintjs/core";
+import { isNumber } from "class-validator";
 import { IPhotoReqJSON } from "~../../src/entity/Photo";
 
 export const AppToaster = Toaster.create({
@@ -43,11 +44,12 @@ export function showPhotoCreateFailToast(f: File, e: string): void {
 }
 
 export function showPhotoUploadJSONFailToast(
-    p: IPhotoReqJSON,
+    p: IPhotoReqJSON | number,
     e: string,
 ): void {
+    const photoMsg = typeof p === "number" ? p : p.hash;
     AppToaster.show({
-        message: `Failed to upload ${p.hash}: ${e}`,
+        message: `Failed to upload ${photoMsg}: ${e}`,
         intent: "danger",
         timeout: 1000,
     });

frontend/src/redux/photos/actions.ts

@@ -14,8 +14,12 @@ export enum PhotoTypes {
     PHOTO_LOAD_SUCCESS = "PHOTO_LOAD_SUCCESS",
     PHOTO_LOAD_FAIL = "PHOTO_LOAD_FAIL",
     PHOTOS_UPLOAD_START = "PHOTOS_UPLOAD",
+    PHOTO_CREATE_QUEUE = "PHOTO_CREATE_QUEUE",
+    PHOTO_CREATE_START = "PHOTO_CREATE_START",
     PHOTO_CREATE_SUCCESS = "PHOTO_CREATE_SUCCESS",
     PHOTO_CREATE_FAIL = "PHOTO_CREATE_FAIL",
+    PHOTO_UPLOAD_QUEUE = "PHOTO_UPLOAD_QUEUE",
+    PHOTO_UPLOAD_START = "PHOTO_UPLOAD_START",
     PHOTO_UPLOAD_SUCCESS = "PHOTO_UPLOAD_SUCCESS",
     PHOTO_UPLOAD_FAIL = "PHOTO_UPLOAD_FAIL",
     PHOTOS_START_FETCHING_SPINNER = "PHOTOS_START_FETCHING_SPINNER",
@@ -60,6 +64,28 @@ export interface IPhotosUploadStartAction extends Action {
     files: FileList;
 }
 
+export interface IPhotoCreateQueue extends Action {
+    type: PhotoTypes.PHOTO_CREATE_QUEUE;
+    file: File;
+}
+
+export interface IPhotoUploadQueue extends Action {
+    type: PhotoTypes.PHOTO_UPLOAD_QUEUE;
+    file: File;
+    id: number;
+}
+
+export interface IPhotoCreateStart extends Action {
+    type: PhotoTypes.PHOTO_CREATE_START;
+    file: File;
+}
+
+export interface IPhotoUploadStart extends Action {
+    type: PhotoTypes.PHOTO_UPLOAD_START;
+    file: File;
+    id: number;
+}
+
 export interface IPhotoUploadSuccessAction extends Action {
     type: PhotoTypes.PHOTO_UPLOAD_SUCCESS;
     photo: IPhotoReqJSON;
@@ -67,13 +93,14 @@ export interface IPhotoUploadSuccessAction extends Action {
 
 export interface IPhotoUploadFailAction extends Action {
     type: PhotoTypes.PHOTO_UPLOAD_FAIL;
-    photo: IPhotoReqJSON;
+    photo: IPhotoReqJSON | number;
     error: string;
 }
 
 export interface IPhotoCreateSuccessAction extends Action {
     type: PhotoTypes.PHOTO_CREATE_SUCCESS;
     photo: IPhotoReqJSON;
+    file: File;
 }
 
 export interface IPhotoCreateFailAction extends Action {
@@ -107,6 +134,22 @@ export interface IPhotosStartFetchingSpinner extends Action {
     type: PhotoTypes.PHOTOS_START_FETCHING_SPINNER;
 }
 
+export function photoCreateQueue(file: File): IPhotoCreateQueue {
+    return { type: PhotoTypes.PHOTO_CREATE_QUEUE, file };
+}
+
+export function photoUploadQueue(file: File, id: number): IPhotoUploadQueue {
+    return { type: PhotoTypes.PHOTO_UPLOAD_QUEUE, file, id };
+}
+
+export function photoCreateStart(file: File): IPhotoCreateStart {
+    return { type: PhotoTypes.PHOTO_CREATE_START, file };
+}
+
+export function photoUploadStart(file: File, id: number): IPhotoUploadStart {
+    return { type: PhotoTypes.PHOTO_UPLOAD_START, file, id };
+}
+
 export function photosLoadStart(): IPhotosLoadStartAction {
     return { type: PhotoTypes.PHOTOS_LOAD_START };
 }
@@ -126,7 +169,7 @@ export function photoUploadSuccess(
 }
 
 export function photoUploadFail(
-    photo: IPhotoReqJSON,
+    photo: IPhotoReqJSON | number,
     error: string,
 ): IPhotoUploadFailAction {
     showPhotoUploadJSONFailToast(photo, error);
@@ -134,7 +177,7 @@ export function photoUploadFail(
 }
 
 export function photoUploadFailWithFile(
-    photo: IPhotoReqJSON,
+    photo: IPhotoReqJSON | number,
     file: File,
     error: string,
 ): IPhotoUploadFailAction {
@@ -144,8 +187,9 @@ export function photoUploadFailWithFile(
 
 export function photoCreateSuccess(
     photo: IPhotoReqJSON,
+    file: File,
 ): IPhotoCreateSuccessAction {
-    return { type: PhotoTypes.PHOTO_CREATE_SUCCESS, photo };
+    return { type: PhotoTypes.PHOTO_CREATE_SUCCESS, photo, file };
 }
 
 export function photoCreateFail(
@@ -172,8 +216,8 @@ export function photoLoadSuccess(
     return { type: PhotoTypes.PHOTO_LOAD_SUCCESS, photo };
 }
 
-export function photoLoadFail(id:number,error: string): IPhotoLoadFailAction {
+export function photoLoadFail(id: number, error: string): IPhotoLoadFailAction {
-    return { type: PhotoTypes.PHOTO_LOAD_FAIL,id, error };
+    return { type: PhotoTypes.PHOTO_LOAD_FAIL, id, error };
 }
 
 export function photoDeleteStart(
@@ -219,4 +263,8 @@ export type PhotoAction =
     | IPhotoDeleteCancelAction
     | IPhotoLoadFailAction
     | IPhotoLoadStartAction
-    | IPhotoLoadSuccessAction;
+    | IPhotoLoadSuccessAction
+    | IPhotoUploadQueue
+    | IPhotoCreateQueue
+    | IPhotoCreateStart
+    | IPhotoUploadStart;

frontend/src/redux/photos/reducer.ts

@@ -18,6 +18,11 @@ export interface IPhotosState {
     overviewFetchingError: string | null;
     overviewFetchingSpinner: boolean;
 
+    photoCreateQueue: File[];
+    photosCreating: number;
+    photoUploadQueue: Record<number, File>;
+    photosUploading: number;
+
     deleteCache: Record<number, IPhotoReqJSON>;
 }
 
@@ -28,6 +33,11 @@ const defaultPhotosState: IPhotosState = {
     overviewFetchingError: null,
     overviewFetchingSpinner: false,
 
+    photoCreateQueue: [],
+    photosCreating: 0,
+    photoUploadQueue: {},
+    photosUploading: 0,
+
     photoStates: {},
 
     deleteCache: {},
@@ -104,31 +114,102 @@ export const photosReducer: Reducer<IPhotosState, PhotoAction> = (
                 photoStates,
             };
         }
-        case PhotoTypes.PHOTO_CREATE_SUCCESS:
+        case PhotoTypes.PHOTO_CREATE_QUEUE: {
+            const { photoCreateQueue } = state;
+            return {
+                ...state,
+                photoCreateQueue: [...photoCreateQueue, action.file],
+            };
+            break;
+        }
+        case PhotoTypes.PHOTO_CREATE_START: {
+            const { photoCreateQueue } = state;
+            const cleanQueue = photoCreateQueue.filter((f) => f != action.file);
+
+            return {
+                ...state,
+                photosCreating: state.photosCreating + 1,
+                photoCreateQueue: cleanQueue,
+            };
+            break;
+        }
+        case PhotoTypes.PHOTO_UPLOAD_START: {
+            const newQueue = state.photoUploadQueue;
+            delete newQueue[action.id];
+
+            return {
+                ...state,
+                photosUploading: state.photosUploading + 1,
+                photoUploadQueue: newQueue,
+            };
+            break;
+        }
+        case PhotoTypes.PHOTO_UPLOAD_QUEUE: {
+            const newQueue = state.photoUploadQueue;
+            newQueue[action.id] = action.file;
+            return {
+                ...state,
+                photoUploadQueue: newQueue,
+            };
+            break;
+        }
+        case PhotoTypes.PHOTO_CREATE_SUCCESS: {
+            const { photoCreateQueue } = state;
+            const cleanQueue = photoCreateQueue.filter((f) => f != action.file);
+
             if (state.photos) {
                 const photos = state.photos;
                 const photosNoDup = photos.filter(
                     (p) => p.id !== action.photo.id,
                 );
                 const updPhotos = [action.photo, ...photosNoDup];
-                return { ...state, photos: updPhotos };
+                return {
+                    ...state,
+                    photos: updPhotos,
+                    photoCreateQueue: cleanQueue,
+                    photosCreating: state.photosCreating - 1,
+                };
             } else {
-                return state;
+                return {
+                    ...state,
+                    photoCreateQueue: cleanQueue,
+                    photosCreating: state.photosCreating - 1,
+                };
             }
-        case PhotoTypes.PHOTO_CREATE_FAIL:
+        }
+        case PhotoTypes.PHOTO_CREATE_FAIL: {
             // TODO: Handle photo create fail
-            return state;
+            const { photoCreateQueue } = state;
-        case PhotoTypes.PHOTO_UPLOAD_SUCCESS:
+            const cleanQueue = photoCreateQueue.filter((f) => f != action.file);
+            return {
+                ...state,
+                photoCreateQueue: cleanQueue,
+                photosCreating: state.photosCreating - 1,
+            };
+        }
+        case PhotoTypes.PHOTO_UPLOAD_SUCCESS: {
+            const newQueue = state.photoUploadQueue;
+            delete newQueue[action.photo.id];
             if (state.photos) {
                 const photos = state.photos;
                 const photosNoDup = photos.filter(
                     (p) => p.id !== action.photo.id,
                 );
                 const updPhotos = [action.photo, ...photosNoDup];
-                return { ...state, photos: updPhotos };
+                return {
+                    ...state,
+                    photos: updPhotos,
+                    photoUploadQueue: newQueue,
+                    photosUploading: state.photosUploading - 1,
+                };
             } else {
-                return state;
+                return {
+                    ...state,
+                    photoUploadQueue: newQueue,
+                    photosUploading: state.photosUploading - 1,
+                };
             }
+        }
         case PhotoTypes.PHOTO_DELETE_START:
             if (state.photos) {
                 const photos = state.photos;
@@ -172,9 +253,20 @@ export const photosReducer: Reducer<IPhotosState, PhotoAction> = (
             return { ...state, deleteCache: delCache, photos };
             break;
         }
-        case PhotoTypes.PHOTO_UPLOAD_FAIL:
+        case PhotoTypes.PHOTO_UPLOAD_FAIL: {
             // TODO: Handle photo upload fail
-            return state;
+            const newQueue = state.photoUploadQueue;
+            if (typeof action.photo === "number") {
+                delete newQueue[action.photo];
+            } else {
+                delete newQueue[action.photo.id];
+            }
+            return {
+                ...state,
+                photoUploadQueue: newQueue,
+                photosUploading: state.photosUploading - 1,
+            };
+        }
         default:
             return state;
     }

frontend/src/redux/photos/sagas.ts

@@ -9,6 +9,7 @@ import {
     takeLatest,
     takeEvery,
     take,
+    select,
 } from "redux-saga/effects";
 import * as SparkMD5 from "spark-md5";
 import {
@@ -23,6 +24,8 @@ import {
     IPhotoLoadStartAction,
     IPhotosUploadStartAction,
     photoCreateFail,
+    photoCreateQueue,
+    photoCreateStart,
     photoCreateSuccess,
     photoDeleteFail,
     photoDeleteSuccess,
@@ -34,6 +37,8 @@ import {
     PhotoTypes,
     photoUploadFail,
     photoUploadFailWithFile,
+    photoUploadQueue,
+    photoUploadStart,
     photoUploadSuccess,
 } from "./actions";
 import { IPhotosNewRespBody } from "~../../src/routes/photos";
@@ -160,57 +165,84 @@ function* photoLoad(action: IPhotoLoadStartAction) {
     }
 }
 
-function* photoUpload(f: File) {
+function* photoCreate() {
-    try {
+    const store = yield select();
-        const hash = yield call(computeChecksumMd5, f);
+    const photosCreating = store.photos.photosCreating;
-        const size = yield call(computeSize, f);
+    if (photosCreating < 2) {
-        const format = f.type;
+        const createQueue = store.photos.photoCreateQueue as File[];
-
+        if (createQueue.length === 0) {
-        const { response, timeout } = yield race({
-            response: call(createPhoto, hash, size, format),
-            timeout: delay(10000),
-        });
-
-        if (timeout) {
-            yield put(photoCreateFail(f, "Timeout"));
             return;
         }
-        if (response.data || response.error === "Photo already exists") {
+        const f = createQueue[0];
-            const photo = (response as IPhotosNewRespBody).data;
+        yield put(photoCreateStart(f));
-            yield put(photoCreateSuccess(photo));
+        try {
+            const hash = yield call(computeChecksumMd5, f);
+            const size = yield call(computeSize, f);
+            const format = f.type;
 
-            try {
+            const { response, timeout } = yield race({
-                const { response, timeout } = yield race({
+                response: call(createPhoto, hash, size, format),
-                    response: call(uploadPhoto, f, photo.id),
+                timeout: delay(10000),
-                    timeout: delay(10000),
+            });
-                });
 
-                if (timeout) {
+            if (timeout) {
-                    yield put(photoUploadFailWithFile(photo, f, "Timeout"));
+                yield put(photoCreateFail(f, "Timeout"));
-                    return;
+                return;
-                }
-                if (response.data) {
-                    const photo = response.data;
-                    yield put(photoUploadSuccess(photo));
-                } else {
-                    yield put(
-                        photoUploadFailWithFile(photo, f, response.error),
-                    );
-                }
-            } catch (e) {
-                yield put(photoUploadFailWithFile(photo, f, "Internal error"));
             }
-        } else {
+            if (response.data || response.error === "Photo already exists") {
-            yield put(photoCreateFail(f, response.error));
+                const photo = (response as IPhotosNewRespBody).data;
+                yield put(photoCreateSuccess(photo, f));
+                yield put(photoUploadQueue(f, photo.id));
+            } else {
+                yield put(photoCreateFail(f, response.error));
+            }
+        } catch (e) {
+            yield put(photoCreateFail(f, "Internal error"));
+        }
+    }
+}
+
+function* photoUpload() {
+    const store = yield select();
+    const photosUploading = store.photos.photosUploading;
+    if (photosUploading < 2) {
+        const createQueue = store.photos.photoUploadQueue as Record<
+            number,
+            File
+        >;
+        if (Object.keys(createQueue).length === 0) {
+            return;
+        }
+        const pId = parseInt(Object.keys(createQueue)[0]);
+        const f = createQueue[pId];
+        yield put(photoUploadStart(f, pId));
+        try {
+            const { response, timeout } = yield race({
+                response: call(uploadPhoto, f, pId),
+                timeout: delay(10000),
+            });
+
+            if (timeout) {
+                yield put(photoUploadFailWithFile(pId, f, "Timeout"));
+                return;
+            }
+            if (response.data) {
+                const photo = response.data;
+                yield put(photoUploadSuccess(photo));
+            } else {
+                yield put(photoUploadFailWithFile(pId, f, response.error));
+            }
+        } catch (e) {
+            yield put(photoUploadFailWithFile(pId, f, "Internal error"));
         }
-    } catch (e) {
-        yield put(photoCreateFail(f, "Internal error"));
     }
 }
 
 function* photosUpload(action: IPhotosUploadStartAction) {
     const files = Array.from(action.files);
-    yield all(files.map((f) => call(photoUpload, f)));
+    for (const file of files) {
+        yield put(photoCreateQueue(file));
+    }
 }
 
 function* photoDelete(action: IPhotoDeleteStartAction) {
@@ -250,5 +282,11 @@ export function* photosSaga() {
         takeLatest(PhotoTypes.PHOTOS_UPLOAD_START, photosUpload),
         takeLatest(PhotoTypes.PHOTO_LOAD_START, photoLoad),
         takeEvery(PhotoTypes.PHOTO_DELETE_START, photoDelete),
+        takeEvery(PhotoTypes.PHOTO_CREATE_QUEUE, photoCreate),
+        takeEvery(PhotoTypes.PHOTO_CREATE_SUCCESS, photoCreate),
+        takeEvery(PhotoTypes.PHOTO_CREATE_FAIL, photoCreate),
+        takeEvery(PhotoTypes.PHOTO_UPLOAD_QUEUE, photoUpload),
+        takeEvery(PhotoTypes.PHOTO_UPLOAD_SUCCESS, photoUpload),
+        takeEvery(PhotoTypes.PHOTO_UPLOAD_FAIL, photoUpload),
     ]);
 }