@ -3,13 +3,12 @@ package web
import (
"bytes"
"encoding/csv"
"encoding/json"
"fmt"
"io"
"log"
"math"
"net/http"
"os"
"path/filepath"
"sort"
"strings"
"sync"
@ -202,59 +201,108 @@ func UploadDocumentsHandler(w http.ResponseWriter, r *http.Request) {
return
}
// Simple multi-upload: use original filenames as display names and default type "1"
// Read and parse file metadata from form values
displayNamesJSON := r . FormValue ( "file_display_names" )
documentTypesJSON := r . FormValue ( "file_document_types" )
isActiveFlagsJSON := r . FormValue ( "file_is_active_flags" )
var displayNames [ ] string
var documentTypes [ ] string
var isActiveFlags [ ] bool
if err := json . Unmarshal ( [ ] byte ( displayNamesJSON ) , & displayNames ) ; err != nil {
log . Printf ( "Error unmarshalling displayNames: %v. JSON: %s" , err , displayNamesJSON )
http . Error ( w , "Error processing file information (display names). Please try again." , http . StatusBadRequest )
return
}
if err := json . Unmarshal ( [ ] byte ( documentTypesJSON ) , & documentTypes ) ; err != nil {
log . Printf ( "Error unmarshalling documentTypes: %v. JSON: %s" , err , documentTypesJSON )
http . Error ( w , "Error processing file information (document types). Please try again." , http . StatusBadRequest )
return
}
if err := json . Unmarshal ( [ ] byte ( isActiveFlagsJSON ) , & isActiveFlags ) ; err != nil {
log . Printf ( "Error unmarshalling isActiveFlags: %v. JSON: %s" , err , isActiveFlagsJSON )
http . Error ( w , "Error processing file information (active flags). Please try again." , http . StatusBadRequest )
return
}
fileHeaders := r . MultipartForm . File [ "documentFiles" ]
if len ( fileHeaders ) == 0 {
http . Error ( w , "No documents selected for upload" , http . StatusBadRequest )
// This case might occur if all files were marked inactive on the client,
// but client still submitted. Or if no files were selected initially.
log . Println ( "No document files received in the request." )
// Depending on desired behavior, could be an error or a silent success if no active files were intended.
// For now, let user know no files were processed if this path is hit.
w . Write ( [ ] byte ( "<p>No files were processed. If you selected files, please ensure some are active for upload.</p>" ) )
return
}
// Validate that the lengths of metadata arrays match the number of files
if len ( fileHeaders ) != len ( displayNames ) || len ( fileHeaders ) != len ( documentTypes ) || len ( fileHeaders ) != len ( isActiveFlags ) {
log . Printf ( "Metadata array length mismatch. Files: %d, Names: %d, Types: %d, ActiveFlags: %d" ,
len ( fileHeaders ) , len ( displayNames ) , len ( documentTypes ) , len ( isActiveFlags ) )
http . Error ( w , "Mismatch in file metadata. Please clear selection and try uploading files again." , http . StatusBadRequest )
return
}
type FileToUploadMetadata struct {
OriginalFilename string
DisplayName string
Type string
TempFile string
File * os . File
Content [ ] byte // Store file content in memory
}
var filesToUpload [ ] FileToUploadMetadata
for _ , fileHeader := range fileHeaders {
for i , fileHeader := range fileHeaders {
if ! isActiveFlags [ i ] {
log . Printf ( "Skipping file %s (original index %d) as it's marked inactive." , fileHeader . Filename , i )
continue // Skip inactive files
}
uploadedFile , err := fileHeader . Open ( )
if err != nil {
log . Printf ( "Error opening uploaded file %s: %v. Skipping." , fileHeader . Filename , err )
log . Printf ( "Error opening uploaded file %s (original index %d): %v. Skipping." , fileHeader . Filename , i , err )
// No need to close uploadedFile here as it wasn't successfully opened.
continue
}
metadata := FileToUploadMetadata {
OriginalFilename : fileHeader . Filename ,
DisplayName : fileHeader . Filename ,
Type : "1" ,
displayName := displayNames [ i ]
docType := documentTypes [ i ]
if strings . TrimSpace ( displayName ) == "" {
displayName = fileHeader . Filename // Fallback to original filename
log . Printf ( "Warning: Empty display name for file %s (original index %d), using original filename." , fileHeader . Filename , i )
}
tempFileHandle , err := os . CreateTemp ( "" , "upload-*" + filepath . Ext ( fileHeader . Filename ) )
if err != nil {
log . Printf ( "Error creating temp file for %s: %v. Skipping." , fileHeader . Filename , err )
uploadedFile . Close ( )
continue
// Basic validation for docType could be added here if necessary, e.g., ensure it's not empty.
if strings . TrimSpace ( docType ) == "" {
docType = "1" // Fallback to a default type
log . Printf ( "Warning: Empty document type for file %s (original index %d, display name %s), using default type '1'." , fileHeader . Filename , i , displayName )
}
if _ , err := io . Copy ( tempFileHandle , uploadedFile ) ; err != nil {
log . Printf ( "Error copying to temp file for %s: %v. Skipping." , fileHeader . Filename , err )
// Read file content into memory
fileBytes , err := io . ReadAll ( uploadedFile )
if err != nil {
log . Printf ( "Error reading content of uploaded file %s (original index %d, display name %s): %v. Skipping." , fileHeader . Filename , i , displayName , err )
uploadedFile . Close ( )
tempFileHandle . Close ( )
os . Remove ( tempFileHandle . Name ( ) )
continue
}
uploadedFile . Close ( )
if _ , err := tempFileHandle . Seek ( 0 , 0 ) ; err != nil {
log . Printf ( "Error seeking temp file for %s: %v. Skipping." , fileHeader . Filename , err )
tempFileHandle . Close ( )
os . Remove ( tempFileHandle . Name ( ) )
continue
uploadedFile . Close ( ) // Close the multipart file handle after reading its content
metadata := FileToUploadMetadata {
OriginalFilename : fileHeader . Filename ,
DisplayName : displayName ,
Type : docType ,
Content : fileBytes ,
}
metadata . TempFile = tempFileHandle . Name ( )
metadata . File = tempFileHandle
filesToUpload = append ( filesToUpload , metadata )
}
activeFilesProcessedCount := len ( filesToUpload )
if activeFilesProcessedCount == 0 {
log . Println ( "No files processed for upload." )
http . Error ( w , "No documents were processed for upload." , http . StatusBadRequest )
log . Println ( "No active files to upload after filtering." )
// Send a user-friendly message back. The resultHTML later will also reflect this.
w . Write ( [ ] byte ( "<p>No active files were selected for upload. Please ensure files are selected and not marked as removed.</p>" ) )
return
}
log . Printf ( "Total active files prepared for upload: %d" , activeFilesProcessedCount )
@ -286,42 +334,27 @@ func UploadDocumentsHandler(w http.ResponseWriter, r *http.Request) {
go func ( jobID string , meta FileToUploadMetadata ) {
defer wg . Done ( )
semaphore <- struct { } { }
defer func ( ) { <- semaphore } ( )
defer func ( ) {
<- semaphore
// No temp file to remove here as content is in memory
} ( )
time . Sleep ( requestDelay )
fileNameForUpload := meta . DisplayName
fileHandleForUpload , err := os . Open ( meta . TempFile )
if err != nil {
log . Printf ( "Goroutine Error: Failed to re-open temp file %s for job %s (uploading as %s): %v" ,
meta . TempFile , jobID , fileNameForUpload , err )
resultsChan <- UploadResult {
JobID : jobID ,
DocName : fileNameForUpload ,
Success : false ,
Error : fmt . Sprintf ( "Error preparing file for upload: %v" , err ) ,
FileSize : 0 ,
}
return
}
defer fileHandleForUpload . Close ( )
fileReaderForUpload := io . NopCloser ( bytes . NewReader ( meta . Content ) )
expectedSize := int64 ( len ( meta . Content ) )
fileInfo , statErr := fileHandleForUpload . Stat ( )
var expectedSize int64
if statErr == nil {
expectedSize = fileInfo . Size ( )
} else {
log . Printf ( "Goroutine Warning: Failed to get file info for %s (original: %s, job %s, uploading as %s): %v" ,
meta . TempFile , meta . OriginalFilename , jobID , fileNameForUpload , statErr )
}
// Error handling for fileReaderForUpload (e.g. if meta.Content is nil) is implicitly handled by API call failures later
// but good to be mindful. For now, assume meta.Content is valid if it reached here.
if len ( jobs ) > 10 {
jitter := time . Duration ( 100 + ( time . Now ( ) . UnixNano ( ) % 400 ) ) * time . Millisecond
time . Sleep ( jitter )
}
sizeTracker := & readCloserWithSize { reader : fileHandle ForUpload , size : 0 }
log . Printf ( "Goroutine Info: Starting to stream file %s (original: %s, uploading as %s, type: %s) to job %s from temp file %s" ,
meta . TempFile , meta . OriginalFilename , fileNameForUpload , meta . Type , jobID , meta . TempFile )
sizeTracker := & readCloserWithSize { reader : fileReaderForUpload , size : 0 }
log . Printf ( "Goroutine Info: Starting to stream in-memory data (original: %s, uploading as %s, type: %s, size: %.2f MB) to job %s" ,
meta . OriginalFilename , fileNameForUpload , meta . Type , float64 ( expectedSize ) / ( 1024 * 1024 ) , jobID )
// Define uploadStart here for per-goroutine timing
uploadStartGoroutine := time . Now ( )
@ -332,13 +365,13 @@ func UploadDocumentsHandler(w http.ResponseWriter, r *http.Request) {
sizeMatch := true
if expectedSize > 0 && math . Abs ( float64 ( expectedSize - fileSizeUploaded ) ) > float64 ( expectedSize ) * 0.05 {
sizeMatch = false
log . Printf ( "Goroutine WARNING: Size mismatch for %s (original: %s, uploaded as %s) to job %s. Expected: %d, Uploaded: %d" ,
meta . TempFile , meta . OriginalFilename , fileNameForUpload , jobID , expectedSize , fileSizeUploaded )
log . Printf ( "Goroutine WARNING: Size mismatch for in-memory data (original: %s, uploaded as %s) to job %s. Expected: %d, Uploaded: %d" ,
meta . OriginalFilename , fileNameForUpload , jobID , expectedSize , fileSizeUploaded )
}
if errUpload != nil {
log . Printf ( "Goroutine Error: Uploading %s (original: %s, as %s) to job %s failed after %v: %v" ,
meta . TempFile , meta . OriginalFilename , fileNameForUpload , jobID , uploadDuration , errUpload )
log . Printf ( "Goroutine Error: Uploading in-memory data (original: %s, as %s) to job %s failed after %v: %v" ,
meta . OriginalFilename , fileNameForUpload , jobID , uploadDuration , errUpload )
resultsChan <- UploadResult {
JobID : jobID ,
DocName : fileNameForUpload ,
@ -347,8 +380,8 @@ func UploadDocumentsHandler(w http.ResponseWriter, r *http.Request) {
FileSize : fileSizeUploaded ,
}
} else if ! sizeMatch {
log . Printf ( "Goroutine Error: Upload of %s (original: %s, as %s) to job %s appears corrupted. API reported success but file sizes mismatch." ,
meta . TempFile , meta . OriginalFilename , fileNameForUpload , jobID )
log . Printf ( "Goroutine Error: Upload of in-memory data (original: %s, as %s) to job %s appears corrupted. API reported success but file sizes mismatch." ,
meta . OriginalFilename , fileNameForUpload , jobID )
resultsChan <- UploadResult {
JobID : jobID ,
DocName : fileNameForUpload ,
@ -357,8 +390,8 @@ func UploadDocumentsHandler(w http.ResponseWriter, r *http.Request) {
FileSize : fileSizeUploaded ,
}
} else {
log . Printf ( "Goroutine Success: Uploaded %s (original: %s, %.2f MB, as %s, type: %s) to job %s in %v" ,
meta . TempFile , meta . OriginalFilename , float64 ( fileSizeUploaded ) / ( 1024 * 1024 ) , fileNameForUpload , meta . Type , jobID , uploadDuration )
log . Printf ( "Goroutine Success: Uploaded in-memory data (original: %s, %.2f MB, as %s, type: %s) to job %s in %v" ,
meta . OriginalFilename , float64 ( fileSizeUploaded ) / ( 1024 * 1024 ) , fileNameForUpload , meta . Type , jobID , uploadDuration )
resultsChan <- UploadResult {
JobID : jobID ,
DocName : fileNameForUpload ,
