diff --git a/lib/gear/metadata.py b/lib/gear/metadata.py index 185f235f..6035bd44 100755 --- a/lib/gear/metadata.py +++ b/lib/gear/metadata.py @@ -70,6 +70,7 @@ def __init__(self, metadata=None, file_path=None): def read_file(self, file_path=None): + print(f'DEBUG: Reading metadata file: {file_path}', file=sys.stderr) """ Reads dataset_metadata.xlsx or dataset_metadata.json into a pandas dataframe @@ -95,7 +96,8 @@ def read_file(self, file_path=None): json_data = {'field': [], 'value': []} with open(file_path) as json_file: - data = ast.literal_eval(json_file.read()) + data = json.loads(json_file.read()) + for d in data: json_data['field'].append(d) json_data['value'].append(data[d]) @@ -273,14 +275,14 @@ def save_to_mysql(self, status=None): is_public = 0 All datasets will save as private. Once the upload is complete, the user can change the dataset to public on the dataset manager. - load_status = 'pending' - All datasets will save as 'pending'. + load_status = 'complete' + All datasets will save as 'complete'. """ if self.metadata is None: raise Exception("No values to evaluate. Please load a metadata file first.") if status is None: - status = 'pending' + status = 'complete' df = self.metadata diff --git a/www/cgi/finalize_uploaded_expression_dataset.cgi b/www/cgi/finalize_uploaded_expression_dataset.cgi new file mode 100755 index 00000000..ca342b4f --- /dev/null +++ b/www/cgi/finalize_uploaded_expression_dataset.cgi @@ -0,0 +1,117 @@ +#!/opt/bin/python3 + +""" +At this point we should have a directory with a JSON file with metadata, a tarball +uploaded by the user, and a status.json file. + +This script does the following: + +- Loads the JSON metadata file and stores it in MySQL +- Migrates the H5AD file to the proper directory +- Migrates the original tarball so it can be downloaded by users + +Returns a status of these steps as JSON data like this: + + +result = { + "success": 1, + "metadata_loaded": 1, + "h5ad_migrated": 1, + "tarball_migrated": 1, + "message": "All steps completed successfully." +} +""" + +import cgi +import json +import os, sys + +lib_path = os.path.abspath(os.path.join('..', '..', 'lib')) +sys.path.append(lib_path) +import geardb + +from gear.metadata import Metadata + +user_upload_file_base = '../uploads/files' +dataset_final_dir = '../datasets' + +result = { + "success": 0, + "metadata_loaded": 0, + "h5ad_migrated": 0, + "tarball_migrated": 0, + "message": "" +} + +def main(): + print('Content-Type: application/json\n\n', flush=True) + + form = cgi.FieldStorage() + share_uid = form.getvalue('share_uid') + session_id = form.getvalue('session_id') + dataset_id = form.getvalue('dataset_uid') + + user = geardb.get_user_from_session_id(session_id) + if user is None: + result['message'] = 'User ID not found. Please log in to continue.' + print(json.dumps(result)) + sys.exit(0) + + dataset_upload_dir = os.path.join(user_upload_file_base, session_id, share_uid) + + # if the upload directory doesn't exist, we can't process the dataset + if not os.path.exists(dataset_upload_dir): + result['message'] = 'Dataset/directory not found.' + print(json.dumps(result)) + sys.exit(0) + + # Load the metadata + metadata_file = os.path.join(dataset_upload_dir, 'metadata.json') + if not os.path.exists(metadata_file): + result['message'] = 'Metadata file not found.' + print(json.dumps(result)) + sys.exit(0) + + with open(metadata_file, 'r') as f: + metadata = json.load(f) + + # Load the metadata into the database + metadata = Metadata(file_path=metadata_file) + try: + metadata.save_to_mysql(status='complete') + result['metadata_loaded'] = 1 + except Exception as e: + result['message'] = 'Error saving metadata to MySQL: {}'.format(str(e)) + print(json.dumps(result)) + sys.exit(0) + + # migrate the H5AD file + h5ad_file = os.path.join(dataset_upload_dir, f'{share_uid}.h5ad') + if not os.path.exists(h5ad_file): + result['message'] = 'H5AD file not found: {}'.format(h5ad_file) + print(json.dumps(result)) + sys.exit(0) + + h5ad_dest = os.path.join(dataset_final_dir, f'{dataset_id}.h5ad') + + try: + os.rename(h5ad_file, h5ad_dest) + result['h5ad_migrated'] = 1 + except Exception as e: + result['message'] = 'Error migrating H5AD file: {}'.format(str(e)) + print(json.dumps(result)) + sys.exit(0) + + + # if we made it this far, all is well, so return success + result['success'] = 1 + result['message'] = 'All steps completed successfully.' + print(json.dumps(result)) + + + + + + +if __name__ == '__main__': + main() diff --git a/www/cgi/get_uploads_in_progress.cgi b/www/cgi/get_uploads_in_progress.cgi index 5b3fe677..c1aba439 100755 --- a/www/cgi/get_uploads_in_progress.cgi +++ b/www/cgi/get_uploads_in_progress.cgi @@ -58,6 +58,7 @@ def main(): result['uploads'].append( { 'share_id': share_id, + 'dataset_id': metadata.get('dataset_uid', ''), 'dataset_type': metadata.get('dataset_type', ''), 'title': metadata.get('title', ''), 'status': 'metadata uploaded', @@ -81,7 +82,9 @@ def main(): if processing_status == 'processing': result['uploads'][-1]['status'] = 'processing' result['uploads'][-1]['load_step'] = 'process-dataset' - + elif processing_status == 'complete': + result['uploads'][-1]['status'] = 'processed' + result['uploads'][-1]['load_step'] = 'finalize-dataset' result['success'] = 1 print(json.dumps(result)) diff --git a/www/js/common.v2.js b/www/js/common.v2.js index 24dfcb5d..780f157f 100644 --- a/www/js/common.v2.js +++ b/www/js/common.v2.js @@ -1282,6 +1282,12 @@ const apiCallsMixin = { const {data} = await axios.post("/cgi/get_user_history_entries.cgi", convertToFormData(payload)); return data; }, + + async finalizeExpressionUpload(formData) { + const payload = new URLSearchParams(formData); + const {data} = await axios.post("/cgi/finalize_uploaded_expression_dataset.cgi", payload); + return data; + }, /** * Retrieves session information. * @returns {Promise} The session information. @@ -1301,7 +1307,6 @@ const apiCallsMixin = { const {data} = await axios.post("/cgi/login.v2.cgi", payload); return data; }, - /** * Parses the metadata file using the provided form data. * @param {FormData} formData - The form data containing the metadata file. @@ -1312,7 +1317,6 @@ const apiCallsMixin = { const {data} = await axios.post("/cgi/upload_expression_metadata.cgi", formData); return data; }, - /** * Renames a dataset collection. * diff --git a/www/js/upload_dataset.js b/www/js/upload_dataset.js index fbbbc255..39cb80a0 100644 --- a/www/js/upload_dataset.js +++ b/www/js/upload_dataset.js @@ -106,6 +106,21 @@ window.onload=function() { } }); + document.getElementById('dataset-finalize-submit').addEventListener('click', (event) => { + event.preventDefault(); + document.getElementById('dataset-finalize-submit').disabled = true; + document.getElementById('finalize-dataset-status-c').classList.remove('is-hidden'); + + finalizeUpload(); + }); + + document.getElementById('dataset-finalize-next-step').addEventListener('click', (event) => { + event.preventDefault(); + + // Move to the next step + stepTo('curate-dataset'); + }); + document.getElementById('metadata-file-input').addEventListener('change', (event) => { // Was a file selected? if (event.target.files.length > 0) { @@ -174,6 +189,42 @@ const checkDatasetProcessingStatus = async () => { } } +const finalizeUpload = async () => { + let formData = new FormData(); + formData.append('share_uid', share_uid); + formData.append('session_id', CURRENT_USER.session_id); + formData.append('dataset_uid', dataset_uid); + + const data = await apiCallsMixin.finalizeExpressionUpload(formData); + + if (data['metadata_loaded']) { + document.getElementById('finalize-storing-metadata').classList.remove('mdi-checkbox-blank-outline'); + document.getElementById('finalize-storing-metadata').classList.add('mdi-checkbox-marked'); + } else { + document.getElementById('finalize-storing-metadata').classList.remove('mdi-checkbox-blank-outline'); + document.getElementById('finalize-storing-metadata').classList.add('mdi-skull-scan'); + } + + if (data['h5ad_migrated']) { + document.getElementById('finalize-migrating-h5ad').classList.remove('mdi-checkbox-blank-outline'); + document.getElementById('finalize-migrating-h5ad').classList.add('mdi-checkbox-marked'); + } else { + document.getElementById('finalize-migrating-h5ad').classList.remove('mdi-checkbox-blank-outline'); + document.getElementById('finalize-migrating-h5ad').classList.add('mdi-skull-scan'); + } + + if (data.success) { + console.log("SUCCESS"); + console.log(data); + document.getElementById('dataset-finalize-next-step').disabled = false; + } else { + console.log("ERROR"); + console.log(data); + document.getElementById('dataset-finalize-status-message').innerText = data.message; + document.getElementById('dataset-finalize-status-message-c').classList.remove('is-hidden'); + } +} + const populateMetadataFormFromFile = async () => { const formData = new FormData(document.getElementById('metadata-upload-form')); const data = await apiCallsMixin.parseMetadataFile(formData); @@ -322,6 +373,7 @@ const loadUploadsInProgress = async () => { data.uploads.forEach((upload) => { let clone = template.content.cloneNode(true); clone.querySelector('tr').dataset.shareId = upload.share_id; + clone.querySelector('tr').dataset.datasetId = upload.dataset_id; clone.querySelector('tr').dataset.loadStep = upload.load_step; clone.querySelector('.submission-share-id').textContent = upload.share_id; @@ -336,6 +388,10 @@ const loadUploadsInProgress = async () => { row.addEventListener('click', (event) => { share_uid = row.dataset.shareId; const step = row.dataset.loadStep; + + if (row.dataset.datasetId) { + dataset_uid = row.dataset.datasetId; + } // Do we want to dynamically load the next step or page refresh for it? // If dynamic we have to reset all the forms. diff --git a/www/upload_dataset.html b/www/upload_dataset.html index e44870a6..8b4be49e 100644 --- a/www/upload_dataset.html +++ b/www/upload_dataset.html @@ -88,7 +88,7 @@

Submissions in progress