diff --git a/internal/handlers/web/documents.go b/internal/handlers/web/documents.go index 93bc7f1..21548ea 100644 --- a/internal/handlers/web/documents.go +++ b/internal/handlers/web/documents.go @@ -173,7 +173,6 @@ func UploadDocumentsHandler(w http.ResponseWriter, r *http.Request) { return } - // Check if the request method is POST if r.Method != http.MethodPost { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return @@ -182,8 +181,6 @@ func UploadDocumentsHandler(w http.ResponseWriter, r *http.Request) { log.Printf("Starting document upload handler with Content-Length: %.2f MB", float64(r.ContentLength)/(1024*1024)) - // Parse the multipart form with a reasonable buffer size - // Files larger than this will be saved to temporary files automatically maxMemory := int64(32 << 20) // 32MB in memory, rest to disk if err := r.ParseMultipartForm(maxMemory); err != nil { log.Printf("Error parsing multipart form: %v", err) @@ -192,14 +189,12 @@ func UploadDocumentsHandler(w http.ResponseWriter, r *http.Request) { } defer r.MultipartForm.RemoveAll() // Clean up temporary files - // Get job numbers from form values jobNumbers := r.FormValue("jobNumbers") if jobNumbers == "" { log.Printf("No job numbers found in hidden 'jobNumbers' input.") http.Error(w, "No job numbers provided", http.StatusBadRequest) return } - log.Printf("Job numbers: %s", jobNumbers) jobs := strings.Split(jobNumbers, ",") if len(jobs) == 0 { @@ -207,118 +202,66 @@ func UploadDocumentsHandler(w http.ResponseWriter, r *http.Request) { return } - // Get the single document type - docType := r.FormValue("documentType") - if docType == "" { - log.Printf("No document type selected") - http.Error(w, "Please select a document type", http.StatusBadRequest) - return - } - log.Printf("Document Type selected: %s", docType) - - // Get the uploaded files from the 'documentFiles' input + // Simple multi-upload: use original filenames as display names and default type "1" fileHeaders := r.MultipartForm.File["documentFiles"] if len(fileHeaders) == 0 { http.Error(w, "No documents selected for upload", http.StatusBadRequest) return } - - // Store file metadata - type FileMetadata struct { - FileName string - Type string - TempFile string // Path to temp file - File *os.File // Open file handle for the temp file + type FileToUploadMetadata struct { + OriginalFilename string + DisplayName string + Type string + TempFile string + File *os.File } - - var filesToUpload []FileMetadata - - // Process each uploaded file + var filesToUpload []FileToUploadMetadata for _, fileHeader := range fileHeaders { - if fileHeader.Filename == "" { - log.Printf("Skipping file header with empty filename.") - continue - } - - // Open the uploaded file uploadedFile, err := fileHeader.Open() if err != nil { - log.Printf("Error opening uploaded file %s: %v", fileHeader.Filename, err) - // Optionally: decide if one error should halt all uploads or just skip this file - continue // Skip this file + log.Printf("Error opening uploaded file %s: %v. Skipping.", fileHeader.Filename, err) + continue } - - // Prepare metadata - metadata := FileMetadata{ - FileName: fileHeader.Filename, - Type: docType, // Use the single document type for all files + metadata := FileToUploadMetadata{ + OriginalFilename: fileHeader.Filename, + DisplayName: fileHeader.Filename, + Type: "1", } - - // Create a temp file for the upload (regardless of size to ensure streaming) - tempFile, err := os.CreateTemp("", "upload-*"+filepath.Ext(fileHeader.Filename)) + tempFileHandle, err := os.CreateTemp("", "upload-*"+filepath.Ext(fileHeader.Filename)) if err != nil { - log.Printf("Error creating temp file for %s: %v", fileHeader.Filename, err) + log.Printf("Error creating temp file for %s: %v. Skipping.", fileHeader.Filename, err) uploadedFile.Close() - continue // Skip this file + continue } - - // Copy the file content to the temp file - bytesCopied, err := io.Copy(tempFile, uploadedFile) - uploadedFile.Close() // Close the original multipart file handle - if err != nil { - log.Printf("Error copying to temp file for %s: %v", fileHeader.Filename, err) - tempFile.Close() // Close the temp file handle - os.Remove(tempFile.Name()) // Remove the partially written temp file - continue // Skip this file + if _, err := io.Copy(tempFileHandle, uploadedFile); err != nil { + log.Printf("Error copying to temp file for %s: %v. Skipping.", fileHeader.Filename, err) + uploadedFile.Close() + tempFileHandle.Close() + os.Remove(tempFileHandle.Name()) + continue } - - log.Printf("Copied %d bytes of %s to temporary file: %s", - bytesCopied, fileHeader.Filename, tempFile.Name()) - - // Seek back to beginning for later reading by upload goroutines - if _, err := tempFile.Seek(0, 0); err != nil { - log.Printf("Error seeking temp file for %s: %v", fileHeader.Filename, err) - tempFile.Close() - os.Remove(tempFile.Name()) - continue // Skip this file + uploadedFile.Close() + if _, err := tempFileHandle.Seek(0, 0); err != nil { + log.Printf("Error seeking temp file for %s: %v. Skipping.", fileHeader.Filename, err) + tempFileHandle.Close() + os.Remove(tempFileHandle.Name()) + continue } - - metadata.TempFile = tempFile.Name() - metadata.File = tempFile // Store the open temp file handle + metadata.TempFile = tempFileHandle.Name() + metadata.File = tempFileHandle filesToUpload = append(filesToUpload, metadata) } - - // Ensure temp files associated with metadata are closed and removed later - defer func() { - log.Println("Running deferred cleanup for temp files...") - for _, fm := range filesToUpload { - if fm.File != nil { - fm.File.Close() - if fm.TempFile != "" { - err := os.Remove(fm.TempFile) - if err != nil && !os.IsNotExist(err) { // Don't log error if file already gone - log.Printf("Error cleaning up temp file %s: %v", fm.TempFile, err) - } else if err == nil { - log.Printf("Cleaned up temp file: %s", fm.TempFile) - } - } - } - } - }() - - if len(filesToUpload) == 0 { - http.Error(w, "No valid documents could be processed for upload", http.StatusBadRequest) + activeFilesProcessedCount := len(filesToUpload) + if activeFilesProcessedCount == 0 { + log.Println("No files processed for upload.") + http.Error(w, "No documents were processed for upload.", http.StatusBadRequest) return } + log.Printf("Total active files prepared for upload: %d", activeFilesProcessedCount) - log.Printf("Total valid files to upload: %d", len(filesToUpload)) - - // Concurrent upload with throttling - // ServiceTrade API allows 30s of availability per minute (approximately 15 requests at 2s each) - const maxConcurrent = 5 // A conservative limit to avoid rate limiting - const requestDelay = 300 * time.Millisecond // Delay between requests + const maxConcurrent = 5 + const requestDelay = 300 * time.Millisecond - // Channel for collecting results type UploadResult struct { JobID string DocName string @@ -328,140 +271,112 @@ func UploadDocumentsHandler(w http.ResponseWriter, r *http.Request) { FileSize int64 } - totalUploads := len(jobs) * len(filesToUpload) + totalUploads := len(jobs) * activeFilesProcessedCount resultsChan := make(chan UploadResult, totalUploads) - - // Create a wait group to track when all uploads are done var wg sync.WaitGroup - - // Create a semaphore channel to limit concurrent uploads semaphore := make(chan struct{}, maxConcurrent) - // Start the upload workers - log.Printf("Starting %d upload workers for %d total uploads (%d jobs x %d files)", - maxConcurrent, totalUploads, len(jobs), len(filesToUpload)) + log.Printf("Starting up to %d concurrent upload workers for %d total uploads (%d jobs x %d active files)", + maxConcurrent, totalUploads, len(jobs), activeFilesProcessedCount) for _, jobID := range jobs { - for _, metadata := range filesToUpload { - // Create a closure capture of the metadata for the goroutine - // This is crucial because the 'metadata' variable in the loop will change - currentMetadata := metadata - + for _, metadataToUpload := range filesToUpload { + currentUploadMetadata := metadataToUpload wg.Add(1) - - // Launch a goroutine for each job+document combination - go func(jobID string, meta FileMetadata) { + go func(jobID string, meta FileToUploadMetadata) { defer wg.Done() - - // Acquire a semaphore slot semaphore <- struct{}{} - defer func() { <-semaphore }() // Release when done - - // Add a small delay to avoid overwhelming the API + defer func() { <-semaphore }() time.Sleep(requestDelay) - // Get the file name to use (original filename) - fileName := meta.FileName - - // Use a fresh reader for the temp file for each upload goroutine - // Re-open the temp file for reading to avoid race conditions on the file pointer - fileHandle, err := os.Open(meta.TempFile) + fileNameForUpload := meta.DisplayName + fileHandleForUpload, err := os.Open(meta.TempFile) if err != nil { - log.Printf("Error re-opening temp file %s for job %s: %v", meta.TempFile, jobID, err) + log.Printf("Goroutine Error: Failed to re-open temp file %s for job %s (uploading as %s): %v", + meta.TempFile, jobID, fileNameForUpload, err) resultsChan <- UploadResult{ JobID: jobID, - DocName: fileName, + DocName: fileNameForUpload, Success: false, - Error: fmt.Sprintf("Error preparing file: %v", err), + Error: fmt.Sprintf("Error preparing file for upload: %v", err), FileSize: 0, } return } - defer fileHandle.Close() // Close this handle when done with this upload + defer fileHandleForUpload.Close() - // Get the expected file size for validation - fileInfo, statErr := fileHandle.Stat() // Stat the newly opened handle + fileInfo, statErr := fileHandleForUpload.Stat() var expectedSize int64 if statErr == nil { expectedSize = fileInfo.Size() } else { - log.Printf("Error getting file info for %s (job %s): %v", fileName, jobID, statErr) - // Continue without size check if stat fails, but log it + log.Printf("Goroutine Warning: Failed to get file info for %s (original: %s, job %s, uploading as %s): %v", + meta.TempFile, meta.OriginalFilename, jobID, fileNameForUpload, statErr) } - // Add jitter delay for large batch uploads (more than 10 jobs) if len(jobs) > 10 { jitter := time.Duration(100+(time.Now().UnixNano()%400)) * time.Millisecond time.Sleep(jitter) } - // Wrap with size tracker - sizeTracker := &readCloserWithSize{reader: fileHandle, size: 0} - - log.Printf("Starting to stream file %s to job %s from temp file %s", fileName, jobID, meta.TempFile) - - // Call ServiceTrade API with the file reader - uploadStart := time.Now() - result, err := session.UploadAttachmentFile(jobID, fileName, meta.Type, sizeTracker) - uploadDuration := time.Since(uploadStart) + sizeTracker := &readCloserWithSize{reader: fileHandleForUpload, size: 0} + log.Printf("Goroutine Info: Starting to stream file %s (original: %s, uploading as %s, type: %s) to job %s from temp file %s", + meta.TempFile, meta.OriginalFilename, fileNameForUpload, meta.Type, jobID, meta.TempFile) - // Get the actual size that was uploaded - fileSize := sizeTracker.Size() + // Define uploadStart here for per-goroutine timing + uploadStartGoroutine := time.Now() + uploadResultData, errUpload := session.UploadAttachmentFile(jobID, fileNameForUpload, meta.Type, sizeTracker) + uploadDuration := time.Since(uploadStartGoroutine) - // Verify the upload size matches the expected file size + fileSizeUploaded := sizeTracker.Size() sizeMatch := true - if expectedSize > 0 && math.Abs(float64(expectedSize-fileSize)) > float64(expectedSize)*0.05 { // Allow 5% tolerance + if expectedSize > 0 && math.Abs(float64(expectedSize-fileSizeUploaded)) > float64(expectedSize)*0.05 { sizeMatch = false - log.Printf("WARNING: Size mismatch for %s to job %s. Expected: %d, Uploaded: %d", - fileName, jobID, expectedSize, fileSize) + log.Printf("Goroutine WARNING: Size mismatch for %s (original: %s, uploaded as %s) to job %s. Expected: %d, Uploaded: %d", + meta.TempFile, meta.OriginalFilename, fileNameForUpload, jobID, expectedSize, fileSizeUploaded) } - if err != nil { - log.Printf("Error uploading %s to job %s after %v: %v", - fileName, jobID, uploadDuration, err) + if errUpload != nil { + log.Printf("Goroutine Error: Uploading %s (original: %s, as %s) to job %s failed after %v: %v", + meta.TempFile, meta.OriginalFilename, fileNameForUpload, jobID, uploadDuration, errUpload) resultsChan <- UploadResult{ JobID: jobID, - DocName: fileName, + DocName: fileNameForUpload, Success: false, - Error: err.Error(), - FileSize: fileSize, + Error: errUpload.Error(), + FileSize: fileSizeUploaded, } } else if !sizeMatch { - // API returned success, but we detected size mismatch - log.Printf("Corrupted upload of %s to job %s detected. API returned success but file sizes don't match.", - fileName, jobID) + log.Printf("Goroutine Error: Upload of %s (original: %s, as %s) to job %s appears corrupted. API reported success but file sizes mismatch.", + meta.TempFile, meta.OriginalFilename, fileNameForUpload, jobID) resultsChan <- UploadResult{ JobID: jobID, - DocName: fileName, + DocName: fileNameForUpload, Success: false, Error: "Upload appears corrupted (file size mismatch)", - FileSize: fileSize, + FileSize: fileSizeUploaded, } } else { - log.Printf("Successfully uploaded %s (%.2f MB) to job %s in %v", - fileName, float64(fileSize)/(1024*1024), jobID, uploadDuration) + log.Printf("Goroutine Success: Uploaded %s (original: %s, %.2f MB, as %s, type: %s) to job %s in %v", + meta.TempFile, meta.OriginalFilename, float64(fileSizeUploaded)/(1024*1024), fileNameForUpload, meta.Type, jobID, uploadDuration) resultsChan <- UploadResult{ JobID: jobID, - DocName: fileName, + DocName: fileNameForUpload, Success: true, - Data: result, - FileSize: fileSize, + Data: uploadResultData, + FileSize: fileSizeUploaded, } } - }(jobID, currentMetadata) // Pass the captured metadata + }(jobID, currentUploadMetadata) } } - // NOTE: The deferred cleanup function for temp files defined earlier will run after this point. - - // Close the results channel when all uploads are done go func() { wg.Wait() close(resultsChan) log.Println("All upload goroutines finished.") }() - // Collect results results := make(map[string][]UploadResult) resultsCount := 0 var totalBytesUploaded int64 @@ -471,11 +386,9 @@ func UploadDocumentsHandler(w http.ResponseWriter, r *http.Request) { log.Printf("Received result %d/%d: Job %s, File %s, Success: %v, Size: %.2f MB", resultsCount, totalUploads, result.JobID, result.DocName, result.Success, float64(result.FileSize)/(1024*1024)) - if result.Success { totalBytesUploaded += result.FileSize } - if _, exists := results[result.JobID]; !exists { results[result.JobID] = []UploadResult{} } @@ -485,10 +398,7 @@ func UploadDocumentsHandler(w http.ResponseWriter, r *http.Request) { log.Printf("All results collected. Total: %d, Total bytes uploaded: %.2f MB", resultsCount, float64(totalBytesUploaded)/(1024*1024)) - // Generate HTML for results var resultHTML bytes.Buffer - - // Count successes and failures var totalSuccess, totalFailure int for _, jobResults := range results { for _, result := range jobResults { @@ -500,39 +410,15 @@ func UploadDocumentsHandler(w http.ResponseWriter, r *http.Request) { } } - // Add summary section resultHTML.WriteString("
All documents were successfully uploaded to ServiceTrade!
") } else if resultsCount == 0 { @@ -540,13 +426,9 @@ func UploadDocumentsHandler(w http.ResponseWriter, r *http.Request) { } else { resultHTML.WriteString("Some documents failed to upload. See details below.
") } + resultHTML.WriteString("No file upload results for this job.
") // More specific message + resultHTML.WriteString("No file upload results for this job.
") } - - resultHTML.WriteString("