diff --git a/src/stitching_scanner.py b/src/stitching_scanner.py index 4442e18..7bcf445 100644 --- a/src/stitching_scanner.py +++ b/src/stitching_scanner.py @@ -207,229 +207,16 @@ class StitchingScanner: return (dx, dy) - def _get_row_start_regions(self, frame: np.ndarray, direction: ScanDirection): - """ - Helper method to calculate the comparison regions (ROIs) for row start alignment. - Returns a dictionary containing the coordinates for Y (vertical) and X (horizontal) checks. - - Returns: - dict or None: Keys 'y_check', 'x_check', 'debug_info'. Returns None if mosaic not ready. - Each check contains: - 'mosaic_roi': (y1, y2, x1, x2), - 'frame_roi': (y1, y2, x1, x2) - """ - if self.mosaic is None: - return None - - mh, mw = self.mosaic.shape[:2] - fh, fw = frame.shape[:2] - - # Calculate layout - # x_offset is where the DOWN transition strips were placed horizontally - x_offset = max(0, mw - self.state.mosaic_init_width) - - # Transition height is how much was added during DOWN transition - transition_height = mh - fh - - regions = { - 'y_check': None, - 'x_check': None, - 'debug_info': { - 'x_offset': x_offset, - 'transition_height': transition_height, - 'mw': mw, 'mh': mh - } - } - - vertical_overlap = min(200, fh // 3) - min_overlap = 50 - - # --------------------------------------------- - # 1. Y Alignment Regions (Top of frame vs Bottom of transition) - # --------------------------------------------- - transition_compare_y_start = transition_height - transition_compare_y_end = max(0, transition_height + vertical_overlap) - - # X range centered on transition strips - x_end = x_offset - x_start = min(x_offset - fw, mw) - compare_width = x_end - x_start - - if transition_compare_y_start >= 0 and compare_width >= min_overlap: - # Mosaic ROI coordinates - m_y1, m_y2 = transition_compare_y_start, transition_compare_y_end - m_x1, m_x2 = x_start, x_end - - # Frame ROI coordinates (Top portion) - f_y1, f_y2 = 0, vertical_overlap - f_x1, f_x2 = 0, compare_width - - regions['y_check'] = { - 'mosaic_roi': (m_y1, m_y2, m_x1, m_x2), - 'frame_roi': (f_y1, f_y2, f_x1, f_x2) - } - - # --------------------------------------------- - # 2. X Alignment Regions - # --------------------------------------------- - horizontal_overlap = min(200, fw // 3) - - # Y range for X check - y_start = 0 - y_end = min(transition_height, fh) - - if direction == ScanDirection.LEFT: - transition_x_start = x_offset - transition_x_end = max(x_offset, x_offset + horizontal_overlap) - - if transition_x_end - transition_x_start >= min_overlap: - # Mosaic ROI - m_y1, m_y2 = y_start, y_end - m_x1, m_x2 = transition_x_start, transition_x_end - - # Frame ROI (Right edge) - f_y1, f_y2 = 0, y_end - f_x1, f_x2 = fw - (transition_x_end - transition_x_start), fw - - regions['x_check'] = { - 'mosaic_roi': (m_y1, m_y2, m_x1, m_x2), - 'frame_roi': (f_y1, f_y2, f_x1, f_x2) - } - - else: # ScanDirection.RIGHT or others - transition_x_start = x_offset - transition_x_end = min(x_offset + horizontal_overlap, mw) - - if transition_x_end - transition_x_start >= min_overlap: - # Mosaic ROI - m_y1, m_y2 = y_start, y_end - m_x1, m_x2 = transition_x_start, transition_x_end - - # Frame ROI (Left edge) - f_y1, f_y2 = 0, y_end - f_x1, f_x2 = 0, transition_x_end - transition_x_start - - regions['x_check'] = { - 'mosaic_roi': (m_y1, m_y2, m_x1, m_x2), - 'frame_roi': (f_y1, f_y2, f_x1, f_x2) - } - - return regions - def _detect_row_start_alignment(self, frame: np.ndarray, direction: ScanDirection) -> AlignmentOffset: """ Detect alignment at the START of a new row. - Uses _get_row_start_regions to determine overlap areas. - """ - offset = AlignmentOffset() - with self._mosaic_lock: - # --- Get the standard regions --- - regions = self._get_row_start_regions(frame, direction) - if not regions: - return offset - - debug_info = regions['debug_info'] - x_offset = debug_info['x_offset'] - transition_height = debug_info['transition_height'] - mw, mh = debug_info['mw'], debug_info['mh'] - fh, fw = frame.shape[:2] - - self.log(f" Row-start alignment: mosaic {mw}x{mh}, frame {fw}x{fh}") - - # ========== DEBUG: Draw borders showing layout (Visuals only) ========== - # WHITE border: Where TRANSITION STRIPS are - cv2.rectangle(self.mosaic, - (x_offset, 0), - (min(x_offset + fw, mw), transition_height), - (255, 255, 255), 3) - - # BLUE line at transition boundary - cv2.line(self.mosaic, (0, transition_height), (mw, transition_height), (255, 0, 0), 3) - - min_overlap = 50 - - # ============================================= - # Step 1: Detect Y alignment - # ============================================= - y_check = regions.get('y_check') - if y_check: - # Extract coordinates - my1, my2, mx1, mx2 = y_check['mosaic_roi'] - fy1, fy2, fx1, fx2 = y_check['frame_roi'] - - # Extract image slices - mosaic_transition_bottom = self.mosaic[my1:my2, mx1:mx2] - frame_compare = frame[fy1:fy2, fx1:fx2] - - # Draw Debug (Cyan) - cv2.rectangle(self.mosaic, (mx1, my1), (mx2, my2), (255, 255, 0), 2) - self.log(f" DEBUG: CYAN border - mosaic Y comparison region") - - # Ensure dimensions match for displacement - min_w = min(frame_compare.shape[1], mosaic_transition_bottom.shape[1]) - min_h = min(frame_compare.shape[0], mosaic_transition_bottom.shape[0]) - - if min_w >= min_overlap and min_h >= min_overlap: - frame_compare = frame_compare[:min_h, :min_w] - mosaic_transition_bottom = mosaic_transition_bottom[:min_h, :min_w] - - # Detect displacement - dx_v, dy_v, conf_v = self._detect_displacement_with_confidence( - mosaic_transition_bottom, frame_compare) - - self.log(f" Row-start Y alignment: dx={dx_v:.1f}, dy={dy_v:.1f}, conf={conf_v:.3f}") - - if conf_v > 0.1: - offset.y_offset = dy_v - offset.confidence = conf_v - - # ============================================= - # Step 2: Detect X alignment - # ============================================= - x_check = regions.get('x_check') - if x_check: - my1, my2, mx1, mx2 = x_check['mosaic_roi'] - fy1, fy2, fx1, fx2 = x_check['frame_roi'] - - mosaic_edge = self.mosaic[my1:my2, mx1:mx2] - frame_edge = frame[fy1:fy2, fx1:fx2] - - # Draw Debug (Yellow) - cv2.rectangle(self.mosaic, (mx1, my1), (mx2, my2), (0, 255, 255), 2) - self.log(f" DEBUG: YELLOW border - mosaic X comparison region") - - min_h = min(mosaic_edge.shape[0], frame_edge.shape[0]) - min_w = min(mosaic_edge.shape[1], frame_edge.shape[1]) - - if min_h >= min_overlap and min_w >= min_overlap: - mosaic_edge = mosaic_edge[:min_h, :min_w] - frame_edge = frame_edge[:min_h, :min_w] - - dx_h, dy_h, conf_h = self._detect_displacement_with_confidence( - mosaic_edge, frame_edge) - - self.log(f" Row-start X alignment: dx={dx_h:.1f}, dy={dy_h:.1f}, conf={conf_h:.3f}") - - if conf_h > 0.1: - offset.x_offset = -dx_h - if conf_h > offset.confidence: - offset.confidence = conf_h - - # Limit maximum adjustment - max_adjust = 80 - offset.x_offset = max(-max_adjust, min(max_adjust, offset.x_offset)) - offset.y_offset = max(-max_adjust, min(max_adjust, offset.y_offset)) - offset.valid = offset.confidence > 0.1 - - return offset - - def _detect_strip_alignment(self, frame: np.ndarray, direction: ScanDirection, - expected_x: int, expected_y: int) -> AlignmentOffset: - """ - Detect alignment offset for a strip. - Attempts to use Row Start regions if available to ensure code path consistency, - otherwise falls back to standard calculated overlap. + After a DOWN transition with prepend, the mosaic layout is: + - Y=0 to Y≈transition_height: TRANSITION STRIPS (placed at x_offset) + - Y≈transition_height to Y=mosaic_height: OLD row 1 content (shifted down) + + The current frame should overlap with the TRANSITION STRIPS, not old row 1. + The camera is at the X position where transition strips were placed. """ offset = AlignmentOffset() @@ -437,102 +224,325 @@ class StitchingScanner: if self.mosaic is None: return offset - # Check if we are in a "Row Start" scenario where we should use the specific regions - # We can try to get the regions; if they return meaningful data, we use them. - # (You might want to add a specific flag or check here if this should only happen - # under specific logic, but retrieving the regions is safe). - rs_regions = self._get_row_start_regions(frame, direction) + mh, mw = self.mosaic.shape[:2] + fh, fw = frame.shape[:2] - # If we found valid X or Y regions in the row-start logic, use those - # This aligns the "Green/Red" boxes with the "Cyan/Yellow" logic - if rs_regions and (rs_regions['x_check'] or rs_regions['y_check']): - self.log(" Strip alignment: Using Row Start regions logic") + # Calculate where transition strips are in the mosaic + # x_offset is where the DOWN transition strips were placed horizontally + x_offset = max(0, mw - self.state.mosaic_init_width) + + # Transition height is how much was added during DOWN transition + # transition_height = mh - original_height_before_transition + # But since row 1 was just fh tall, transition_height = mh - fh + transition_height = mh - fh + + self.log(f" Row-start alignment: mosaic {mw}x{mh}, frame {fw}x{fh}") + self.log(f" Transition strips at: X={x_offset}, Y=0 to Y={transition_height}") + self.log(f" Old row 1 shifted to: Y={transition_height} to Y={mh}") + + # ========== DEBUG: Draw borders showing layout ========== + # WHITE border: Where TRANSITION STRIPS are (this is what we should compare with) + cv2.rectangle(self.mosaic, + (x_offset, 0), + (min(x_offset + fw, mw), transition_height), + (255, 255, 255), 3) # WHITE - transition strips location + self.log(f" DEBUG: WHITE border - TRANSITION STRIPS at X={x_offset}:{min(x_offset + fw, mw)}, Y=0:{transition_height}") + + # BLUE line at transition boundary (where transition ends and old row 1 begins) + cv2.line(self.mosaic, (0, transition_height), (mw, transition_height), (255, 0, 0), 3) + self.log(f" DEBUG: BLUE line at Y={transition_height} (transition/row1 boundary)") + + vertical_overlap = min(200, fh // 3) + min_overlap = 50 + + # ============================================= + # Step 1: Detect Y alignment + # ============================================= + # Compare frame's TOP with the BOTTOM of transition strips + # Frame's top portion overlaps with mosaic's transition bottom + # + # Transition strips are at Y=0 to Y=transition_height + # So transition bottom is around Y=(transition_height - overlap) to Y=transition_height + + transition_compare_y_start = transition_height + transition_compare_y_end = max(0, transition_height + vertical_overlap) + + # Frame's TOP should overlap with transition BOTTOM + frame_top = frame[:vertical_overlap, :] + + # Get the X range - centered on where transition strips are + x_end = x_offset + x_start = min(x_offset - fw, mw) + compare_width = x_end - x_start + + if transition_compare_y_start >= 0 and compare_width >= min_overlap: + # Mosaic region: bottom of transition strips + mosaic_transition_bottom = self.mosaic[transition_compare_y_start:transition_compare_y_end, + x_start:x_end] - # We can accumulate offsets from both checks if they exist - # Or prioritize one. Row start logic does both. + # Frame region: top portion (what overlaps with transition) + frame_compare = frame_top[:, :compare_width] - # 1. Check Vertical (Y) - if rs_regions['y_check']: - my1, my2, mx1, mx2 = rs_regions['y_check']['mosaic_roi'] - fy1, fy2, fx1, fx2 = rs_regions['y_check']['frame_roi'] + # ========== DEBUG: Save frame with comparison region marked ========== + debug_frame = frame.copy() + cv2.rectangle(debug_frame, (0, 0), (compare_width, vertical_overlap), + (255, 255, 0), 2) # CYAN - frame's top region used for Y comparison + self.log(f" DEBUG: Frame Y comparison region: X=0:{compare_width}, Y=0:{vertical_overlap}") + try: + cv2.imwrite('/tmp/debug_frame_row2_start.png', debug_frame) + except: + pass + + # ========== DEBUG: Draw CYAN border on mosaic for Y comparison region ========== + cv2.rectangle(self.mosaic, + (x_start, transition_compare_y_start), + (x_end, transition_compare_y_end), + (255, 255, 0), 2) # CYAN - mosaic comparison region for Y + self.log(f" DEBUG: CYAN border - mosaic Y comparison region X={x_start}:{x_end}, Y={transition_compare_y_start}:{transition_compare_y_end}") + + # MAGENTA border: Where frame is EXPECTED to be placed (at transition position) + cv2.rectangle(self.mosaic, + (x_start, 0), + (x_end, fh), + (255, 0, 255), 2) # MAGENTA - expected frame position + self.log(f" DEBUG: MAGENTA border - expected frame position X={x_start}:{x_end}, Y=0:{fh}") + + min_w = min(frame_compare.shape[1], mosaic_transition_bottom.shape[1]) + min_h = min(frame_compare.shape[0], mosaic_transition_bottom.shape[0]) + + if min_w >= min_overlap and min_h >= min_overlap: + frame_compare = frame_compare[:min_h, :min_w] + mosaic_transition_bottom = mosaic_transition_bottom[:min_h, :min_w] - mosaic_region = self.mosaic[my1:my2, mx1:mx2] - frame_region = frame[fy1:fy2, fx1:fx2] + # ========== DEBUG: Save the comparison regions as images ========== + try: + cv2.imwrite('/tmp/debug_frame_top_region.png', frame_compare) + cv2.imwrite('/tmp/debug_mosaic_transition_region.png', mosaic_transition_bottom) + self.log(f" DEBUG: Saved comparison regions to /tmp/debug_*.png") + except: + pass - # Ensure same size - h, w = min(mosaic_region.shape[:2]), min(frame_region.shape[:2]) - if h > 10 and w > 10: - dx, dy, conf = self._detect_displacement_with_confidence( - mosaic_region[:h, :w], frame_region[:h, :w]) + # Detect displacement + dx_v, dy_v, conf_v = self._detect_displacement_with_confidence( + mosaic_transition_bottom, frame_compare) + + self.log(f" Row-start Y alignment: dx={dx_v:.1f}, dy={dy_v:.1f}, conf={conf_v:.3f}") + self.log(f" Compared frame[0:{vertical_overlap}] with mosaic[{transition_compare_y_start}:{transition_compare_y_end}]") + + if conf_v > 0.1: + offset.y_offset = dy_v + offset.confidence = conf_v + + # ============================================= + # Step 2: Detect X alignment + # ============================================= + horizontal_overlap = min(200, fw // 3) + + if direction == ScanDirection.LEFT: + # For LEFT scan: frame starts at the transition X position + # Compare frame's RIGHT edge with mosaic's transition strip RIGHT edge + + transition_x_start = x_offset + transition_x_end = max(x_offset, x_offset + horizontal_overlap) + + # Y range: within the transition strip area + y_start = 0 + y_end = min(transition_height, fh) + + if transition_x_end - transition_x_start >= min_overlap: + mosaic_edge = self.mosaic[y_start:y_end, transition_x_start:transition_x_end] + frame_edge = frame[:y_end, fw - (transition_x_end - transition_x_start):fw] + + # ========== DEBUG: Draw YELLOW border for X comparison region ========== + cv2.rectangle(self.mosaic, + (transition_x_start, y_start), + (transition_x_end, y_end), + (0, 255, 255), 2) # YELLOW - mosaic comparison region for X + self.log(f" DEBUG: YELLOW border - mosaic X comparison region X={transition_x_start}:{transition_x_end}, Y={y_start}:{y_end}") + + min_h = min(mosaic_edge.shape[0], frame_edge.shape[0]) + min_w = min(mosaic_edge.shape[1], frame_edge.shape[1]) + + if min_h >= min_overlap and min_w >= min_overlap: + mosaic_edge = mosaic_edge[:min_h, :min_w] + frame_edge = frame_edge[:min_h, :min_w] - if conf > 0.1: - offset.y_offset = dy - if conf > offset.confidence: - offset.confidence = conf - - # 2. Check Horizontal (X) - if rs_regions['x_check']: - my1, my2, mx1, mx2 = rs_regions['x_check']['mosaic_roi'] - fy1, fy2, fx1, fx2 = rs_regions['x_check']['frame_roi'] - - mosaic_region = self.mosaic[my1:my2, mx1:mx2] - frame_region = frame[fy1:fy2, fx1:fx2] - - h, w = min(mosaic_region.shape[:2]), min(frame_region.shape[:2]) - if h > 10 and w > 10: - dx, dy, conf = self._detect_displacement_with_confidence( - mosaic_region[:h, :w], frame_region[:h, :w]) + dx_h, dy_h, conf_h = self._detect_displacement_with_confidence( + mosaic_edge, frame_edge) - if conf > 0.1: - # Note: Row start logic specifically inverts X sometimes depending on logic - # but here we usually want pure displacement. - # If row_start_alignment did 'offset.x_offset = -dx_h', verify directions. - # Assuming standard displacement matches: - offset.x_offset = -dx - if conf > offset.confidence: - offset.confidence = conf + self.log(f" Row-start X alignment: dx={dx_h:.1f}, dy={dy_h:.1f}, conf={conf_h:.3f}") + + if conf_h > 0.1: + offset.x_offset = -dx_h + if conf_h > offset.confidence: + offset.confidence = conf_h + else: + # For RIGHT scan at row start (similar logic but for left edge) + transition_x_start = x_offset + transition_x_end = min(x_offset + horizontal_overlap, mw) - offset.valid = offset.confidence > 0.1 + y_start = 0 + y_end = min(transition_height, fh) + + if transition_x_end - transition_x_start >= min_overlap: + mosaic_edge = self.mosaic[y_start:y_end, transition_x_start:transition_x_end] + frame_edge = frame[:y_end, :transition_x_end - transition_x_start] + + cv2.rectangle(self.mosaic, + (transition_x_start, y_start), + (transition_x_end, y_end), + (0, 255, 255), 2) # YELLOW + self.log(f" DEBUG: YELLOW border - mosaic X comparison region X={transition_x_start}:{transition_x_end}, Y={y_start}:{y_end}") + + min_h = min(mosaic_edge.shape[0], frame_edge.shape[0]) + min_w = min(mosaic_edge.shape[1], frame_edge.shape[1]) + + if min_h >= min_overlap and min_w >= min_overlap: + mosaic_edge = mosaic_edge[:min_h, :min_w] + frame_edge = frame_edge[:min_h, :min_w] + + dx_h, dy_h, conf_h = self._detect_displacement_with_confidence( + mosaic_edge, frame_edge) + + self.log(f" Row-start X alignment: dx={dx_h:.1f}, dy={dy_h:.1f}, conf={conf_h:.3f}") + + if conf_h > 0.1: + offset.x_offset = -dx_h + if conf_h > offset.confidence: + offset.confidence = conf_h + + # Limit maximum adjustment + max_adjust = 80 + if abs(offset.x_offset) > max_adjust: + self.log(f" Limiting X offset from {offset.x_offset:.1f} to ±{max_adjust}") + offset.x_offset = max(-max_adjust, min(max_adjust, offset.x_offset)) + if abs(offset.y_offset) > max_adjust: + self.log(f" Limiting Y offset from {offset.y_offset:.1f} to ±{max_adjust}") + offset.y_offset = max(-max_adjust, min(max_adjust, offset.y_offset)) + + offset.valid = offset.confidence > 0.1 + + if offset.valid: + self.log(f" Row-start alignment FINAL: X={offset.x_offset:.1f}, Y={offset.y_offset:.1f}, conf={offset.confidence:.3f}") + + return offset + + def _detect_strip_alignment(self, frame: np.ndarray, direction: ScanDirection, + expected_x: int, expected_y: int) -> AlignmentOffset: + """ + Detect alignment offset for a strip by comparing the current frame + with the expected overlap region of the mosaic. + + This provides continuous correction for gear slippage during scanning. + + Args: + frame: Current camera frame + direction: Scan direction + expected_x: Expected X position in mosaic + expected_y: Expected Y position in mosaic + + Returns: + AlignmentOffset with X/Y correction needed + """ + offset = AlignmentOffset() + + with self._mosaic_lock: + if self.mosaic is None: return offset - - # --- FALLBACK: Original Strip Alignment Logic --- - # If _get_row_start_regions returned nothing useful (e.g. not in that zone), - # proceed with standard overlap logic based on expected_x/y mh, mw = self.mosaic.shape[:2] fh, fw = frame.shape[:2] - # ... [Rest of original _detect_strip_alignment logic goes here] ... - # ... (Clamp expected positions, switch on direction, etc) ... + # Clamp expected positions + expected_y = max(0, min(expected_y, mh - fh)) + expected_x = max(0, min(expected_x, mw - fw)) - # (Included for completeness of the example flow) - max_overlap = 250 - min_overlap = 40 + # Increased overlap for better detection + max_overlap = 250 # Increased from 200 + min_overlap = 40 # Increased from 30 - # [Standard logic setup...] if direction == ScanDirection.RIGHT: + # We're appending to the right + # Compare left portion of frame with right edge of mosaic overlap_width = min(fw // 2, mw - expected_x, max_overlap) - if overlap_width < min_overlap: return offset + + if overlap_width < min_overlap: + return offset + + # Extract regions mosaic_region = self.mosaic[expected_y:expected_y + fh, mw - overlap_width:mw] frame_region = frame[:, :overlap_width] - # ... [Handle other directions] ... - else: - return offset # Simplified for brevity - - # Execute standard detection + + elif direction == ScanDirection.LEFT: + # We're placing within existing mosaic, moving left + # Compare right portion of frame with mosaic at expected position + overlap_width = min(fw // 2, mw - expected_x, max_overlap) + + if overlap_width < min_overlap: + return offset + + # The frame's right edge should align with mosaic at expected_x + fw + mosaic_x_end = min(expected_x + fw, mw) + mosaic_x_start = max(mosaic_x_end - overlap_width, 0) + actual_overlap = mosaic_x_end - mosaic_x_start + + if actual_overlap < min_overlap: + return offset + + mosaic_region = self.mosaic[expected_y:expected_y + fh, mosaic_x_start:mosaic_x_end] + frame_region = frame[:, fw - actual_overlap:] + + elif direction == ScanDirection.DOWN: + # We're appending below + # Compare top portion of frame with bottom edge of mosaic + overlap_height = min(fh // 2, mh - expected_y, max_overlap) + + if overlap_height < min_overlap: + return offset + + mosaic_region = self.mosaic[mh - overlap_height:mh, expected_x:expected_x + fw] + frame_region = frame[:overlap_height, :] + + else: # UP + # Compare bottom portion of frame with top edge of mosaic + overlap_height = min(fh // 2, expected_y, max_overlap) + + if overlap_height < min_overlap: + return offset + + mosaic_region = self.mosaic[:overlap_height, expected_x:expected_x + fw] + frame_region = frame[fh - overlap_height:, :] + + # Ensure regions have the same size min_h = min(mosaic_region.shape[0], frame_region.shape[0]) min_w = min(mosaic_region.shape[1], frame_region.shape[1]) + + if min_h < min_overlap or min_w < min_overlap: + self.log(f"Strip alignment: overlap too small ({min_w}x{min_h})") + return offset + mosaic_region = mosaic_region[:min_h, :min_w] frame_region = frame_region[:min_h, :min_w] - + + # Detect displacement with confidence dx, dy, confidence = self._detect_displacement_with_confidence(mosaic_region, frame_region) + # Sanity check - reject large displacements + max_adjust = 500 # Max pixels to adjust + if abs(dx) > max_adjust or abs(dy) > max_adjust: + self.log(f"Strip alignment: displacement too large ({dx:.1f}, {dy:.1f}), ignoring") + return offset + offset.x_offset = dx offset.y_offset = dy offset.confidence = confidence - offset.valid = confidence > 0.1 + offset.valid = confidence > 0.1 # Require minimum confidence - return offset + if offset.valid: + self.log(f" Strip alignment: X={dx:.1f}, Y={dy:.1f}, conf={confidence:.3f}") + + return offset + # ========================================================================= # Mosaic Building # =========================================================================