Vision Foundation Models: Implementation And Business Applications

!pip instal torch torchvision transformers timm pillow matplotlib opencv-python tensorflow-hub tensorflow !pip instal huggingface_hub sentence-transformers ftfy regex tqdm !pip instal accelerate

import torch print(f"PyTorch version: {torch.__version__}") print(f"CUDA available: {torch.cuda.is_available()}") if torch.cuda.is_available(): print(f"CUDA device: {torch.cuda.get_device_name(0)}")

import torch from PIL import Image import requests from transformers import CLIPProcessor, CLIPModel import matplotlib.pyplot arsenic plt import numpy arsenic np # Load exemplary and processor model_id = "openai/clip-vit-base-patch32" model = CLIPModel.from_pretrained(model_id) processor = CLIPProcessor.from_pretrained(model_id) # Function to get image embeddings def get_clip_image_embedding(image_path): image = Image.open(image_path) if isinstance(image_path, str) other image_path inputs = processor(images=image, return_tensors="pt") pinch torch.no_grad(): image_features = model.get_image_features(**inputs) return image_features # Function to execute zero-shot classification def classify_image_with_clip(image_path, categories): image = Image.open(image_path) if isinstance(image_path, str) other image_path inputs = processor( text=categories, images=image, return_tensors="pt", padding=True ) pinch torch.no_grad(): outputs = model(**inputs) logits_per_image = outputs.logits_per_image probs = logits_per_image.softmax(dim=1) # Return dict of categories and probabilities return {categories[i]: probs[0][i].item() for one successful range(len(categories))} # Example: Product categorization url = "https://images.unsplash.com/photo-1542291026-7eec264c27ff?q=80&w=1470&auto=format&fit=crop" image = Image.open(requests.get(url, stream=True).raw) product_categories = [ "sneakers", "formal shoes", "sandals", "boots", "sports equipment", "casual wear", "luxury item" ] results = classify_image_with_clip(image, product_categories) # Sort results by probability sorted_results = dict(sorted(results.items(), key=lambda x: x[1], reverse=True)) # Display nan image and classification results plt.figure(figsize=(12, 6)) # Plot nan image connected nan left plt.subplot(1, 2, 1) plt.imshow(np.array(image)) plt.title("Input Image") plt.axis("off") # Plot nan classification results connected nan right plt.subplot(1, 2, 2) categories = list(sorted_results.keys()) scores = list(sorted_results.values()) y_pos = np.arange(len(categories)) plt.barh(y_pos, scores, align="center") plt.yticks(y_pos, categories) plt.xlabel("Probability") plt.title("CLIP Classification Results") plt.tight_layout() plt.show() # Also people results to console print("Classification Results:") for category, people successful sorted_results.items(): print(f"{category}: {score:.4f}")

import torch import torchvision.transforms arsenic T from PIL import Image import numpy arsenic np import matplotlib.pyplot arsenic plt from torch.nn import functional arsenic F import requests from io import BytesIO # Load DINOv2 model dinov2_vits14 = torch.hub.load('facebookresearch/dinov2', 'dinov2_vits14') dinov2_vits14.eval() # Preprocess images for DINOv2 transform = T.Compose([ T.Resize(256), T.CenterCrop(224), T.ToTensor(), T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) # Function to extract features def extract_dinov2_features(image_path): image = Image.open(image_path).convert('RGB') if isinstance(image_path, str) other image_path img_tensor = transform(image).unsqueeze(0) pinch torch.no_grad(): features = dinov2_vits14(img_tensor) return features # Function to compute similarity betwixt images def compute_similarity(img1_path, img2_path): feat1 = extract_dinov2_features(img1_path) feat2 = extract_dinov2_features(img2_path) # Normalize features feat1 = F.normalize(feat1, dim=1) feat2 = F.normalize(feat2, dim=1) # Compute cosine similarity similarity = torch.mm(feat1, feat2.transpose(0, 1)).item() return similarity # Function to download image from URL def download_image(url): consequence = requests.get(url, stream=True) return Image.open(BytesIO(response.content)).convert('RGB') # Function to visualize image brace pinch similarity score def visualize_similarity(img1_path, img2_path, title=None): # Load images if img1_path.startswith(('http://', 'https://')): img1 = download_image(img1_path) else: img1 = Image.open(img1_path).convert('RGB') if img2_path.startswith(('http://', 'https://')): img2 = download_image(img2_path) else: img2 = Image.open(img2_path).convert('RGB') # Compute similarity similarity = compute_similarity(img1, img2) # Create fig for visualization fig, axes = plt.subplots(1, 2, figsize=(12, 6)) # Display images axes[0].imshow(np.array(img1)) axes[0].set_title("Image 1") axes[0].axis("off") axes[1].imshow(np.array(img2)) axes[1].set_title("Image 2") axes[1].axis("off") # Add similarity people arsenic fig title fig_title = f"Similarity Score: {similarity:.4f}" if title: fig_title = f"{title}n{fig_title}" fig.suptitle(fig_title, fontsize=16) plt.tight_layout() plt.show() return similarity # Example: Use nonstop URLs alternatively of downloading files first # Sample sneaker images from Unsplash url1 = "https://images.unsplash.com/photo-1560769629-975ec94e6a86?w=500" # Red sneaker url2 = "https://images.unsplash.com/photo-1600185365926-3a2ce3cdb9eb?w=500" # White sneaker url3 = "https://images.unsplash.com/photo-1491553895911-0055eca6402d?w=500" # Another sneaker # Visualize pairs pinch similarity scores print("Comparing Product 1 and Product 2:") similarity_1_2 = visualize_similarity(url1, url2, "Red Sneaker vs White Sneaker") print("nComparing Product 1 and Product 3:") similarity_1_3 = visualize_similarity(url1, url3, "Red Sneaker vs Another Sneaker") print("nComparing Product 2 and Product 3:") similarity_2_3 = visualize_similarity(url2, url3, "White Sneaker vs Another Sneaker") # Print summary of each similarities print("nSummary of Similarity Scores:") print(f"Similarity betwixt merchandise 1 and 2: {similarity_1_2:.4f}") print(f"Similarity betwixt merchandise 1 and 3: {similarity_1_3:.4f}") print(f"Similarity betwixt merchandise 2 and 3: {similarity_2_3:.4f}")

# Install required libraries for SAM !pip instal git+https://github.com/facebookresearch/segment-anything.git import torch import numpy arsenic np import matplotlib.pyplot arsenic plt from segment_anything import sam_model_registry, SamPredictor import cv2 from PIL import Image import requests # Download SAM checkpoint !wget -q https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth # Load SAM model sam = sam_model_registry["vit_h"](checkpoint="sam_vit_h_4b8939.pth") device = "cuda" if torch.cuda.is_available() other "cpu" sam.to(device) predictor = SamPredictor(sam) # Function to execute automatic segmentation def segment_image(image_path): # Load image image = cv2.imread(image_path) image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # Set image for SAM predictor.set_image(image_rgb) # Generate automatic masks masks, scores, logits = predictor.predict( point_coords=None, point_labels=None, multimask_output=True, box=None ) return image_rgb, masks, scores # Function to visualize segmentation results def visualize_segmentation(image, masks, scores, limit=5): plt.figure(figsize=(15, 10)) # Display original image plt.subplot(1, limit+1, 1) plt.imshow(image) plt.title("Original Image") plt.axis('off') # Display apical masks top_indices = np.argsort(scores)[-limit:][::-1] for i, idx successful enumerate(top_indices): plt.subplot(1, limit+1, i+2) plt.imshow(image) plt.imshow(masks[idx], alpha=0.7, cmap='jet') plt.title(f"Mask {i+1}nScore: {scores[idx]:.3f}") plt.axis('off') plt.tight_layout() plt.show() # Example: Product segmentation for e-commerce !wget -q -O product_image.jpg "https://images.unsplash.com/photo-1525966222134-fcfa99b8ae77?w=800" image_rgb, masks, scores = segment_image("product_image.jpg") visualize_segmentation(image_rgb, masks, scores) # Business application: Calculate precise merchandise measurements def calculate_object_dimensions(mask): # Find contours successful nan mask contours, _ = cv2.findContours((mask * 255).astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) if not contours: return None # Get nan largest contour largest_contour = max(contours, key=cv2.contourArea) # Get bounding rectangle x, y, w, h = cv2.boundingRect(largest_contour) # Calculate facet ratio aspect_ratio = w / h # Calculate area successful pixels area_pixels = cv2.contourArea(largest_contour) return { 'width': w, 'height': h, 'aspect_ratio': aspect_ratio, 'area_pixels': area_pixels } # Apply to nan highest scoring mask best_mask_idx = np.argmax(scores) dimensions = calculate_object_dimensions(masks[best_mask_idx]) print("Product Dimensions:") print(f"Width: {dimensions['width']} pixels") print(f"Height: {dimensions['height']} pixels") print(f"Aspect Ratio: {dimensions['aspect_ratio']:.2f}") print(f"Area: {dimensions['area_pixels']} quadrate pixels")

from transformers import Blip2Processor, Blip2ForConditionalGeneration import torch from PIL import Image import requests import matplotlib.pyplot arsenic plt import numpy arsenic np from io import BytesIO # Load BLIP-2 model processor = Blip2Processor.from_pretrained("Salesforce/blip2-opt-2.7b") model = Blip2ForConditionalGeneration.from_pretrained("Salesforce/blip2-opt-2.7b", torch_dtype=torch.float16) if torch.cuda.is_available(): exemplary = model.to("cuda") # Function to download image from URL def download_image(url): consequence = requests.get(url, stream=True) return Image.open(BytesIO(response.content)).convert('RGB') # Function for image captioning def generate_caption(image_path): # Load image from way aliases URL if isinstance(image_path, str): if image_path.startswith(('http://', 'https://')): image = download_image(image_path) else: image = Image.open(image_path).convert('RGB') else: image = image_path inputs = processor(images=image, return_tensors="pt") if torch.cuda.is_available(): inputs = {k: v.to("cuda") for k, v successful inputs.items()} generated_ids = model.generate(**inputs, max_new_tokens=50) generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0].strip() return generated_text # Function for ocular mobility answering def visual_qa(image_path, question): # Load image from way aliases URL if isinstance(image_path, str): if image_path.startswith(('http://', 'https://')): image = download_image(image_path) else: image = Image.open(image_path).convert('RGB') else: image = image_path # FIX: Properly format nan mobility for nan model # BLIP-2 needs a circumstantial punctual format for QA punctual = f"Question: {question} Answer:" inputs = processor(images=image, text=prompt, return_tensors="pt") if torch.cuda.is_available(): inputs = {k: v.to("cuda") for k, v successful inputs.items()} generated_ids = model.generate( **inputs, max_new_tokens=30, do_sample=False # Use greedy decoding for much precise answers ) reply = processor.batch_decode(generated_ids, skip_special_tokens=True)[0].strip() # Remove nan punctual portion from nan answer reply = answer.replace(prompt, "").strip() return answer # Function to visualize image pinch caption and QA def visualize_product_analysis(image_path, questions=None): # Load image if isinstance(image_path, str): if image_path.startswith(('http://', 'https://')): image = download_image(image_path) else: image = Image.open(image_path).convert('RGB') else: image = image_path # Generate caption caption = generate_caption(image) # Default questions if nary provided if questions is None: questions = [ "What colour is this product?", "What worldly is this merchandise made of?", "What is nan target demographic for this product?", "What is simply a cardinal characteristic of this product?" ] # Get answers answers = [] for mobility successful questions: reply = visual_qa(image, question) answers.append((question, answer)) # Create visualization plt.figure(figsize=(12, 10)) # Display image plt.subplot(2, 1, 1) plt.imshow(np.array(image)) plt.title("Product Image", fontsize=14) plt.axis('off') # Display caption and Q&A plt.subplot(2, 1, 2) plt.axis('off') text_content = f"Generated Description: {caption}nn" text_content += "Product Analysis:n" for q, a successful answers: text_content += f"Q: {q}nA: {a}nn" plt.text(0.01, 0.99, text_content, transform=plt.gca().transAxes, fontsize=12, verticalalignment='top', wrap=True) plt.tight_layout() plt.show() return caption, answers # Business application: Automated merchandise listing def create_product_listing(image_path): # Load image if isinstance(image_path, str): if image_path.startswith(('http://', 'https://')): image = download_image(image_path) else: image = Image.open(image_path).convert('RGB') else: image = image_path # Get basal caption caption = generate_caption(image) # Extract merchandise attributes pinch much circumstantial prompting colour = visual_qa(image, "What colors are visible successful this product?") worldly = visual_qa(image, "What worldly does this merchandise look to beryllium made of?") use_case = visual_qa(image, "What would beryllium nan main usage lawsuit for this product?") unique_features = visual_qa(image, "What are immoderate unsocial aliases notable features of this product?") # Create system listing listing = { "title": caption, "attributes": { "color": color, "material": material, "primary_use": use_case, "unique_features": unique_features } } # Visualize nan listing plt.figure(figsize=(14, 10)) # Display image plt.subplot(1, 2, 1) plt.imshow(np.array(image)) plt.title("Product Image", fontsize=14) plt.axis('off') # Display listing details plt.subplot(1, 2, 2) plt.axis('off') listing_text = f"PRODUCT LISTINGnn" listing_text += f"Title: {listing['title']}nn" listing_text += "Product Attributes:n" for attr, worth successful listing['attributes'].items(): listing_text += f"{attr.replace('_', ' ').title()}: {value}n" plt.text(0.01, 0.99, listing_text, transform=plt.gca().transAxes, fontsize=12, verticalalignment='top') plt.tight_layout() plt.show() return listing # Function for trading contented analysis def analyze_marketing_content(image_path): # Load image if isinstance(image_path, str): if image_path.startswith(('http://', 'https://')): image = download_image(image_path) else: image = Image.open(image_path).convert('RGB') else: image = image_path # Marketing-specific questions marketing_questions = [ "What emotions does this image evoke?", "What marque values are communicated successful this image?", "What target assemblage would this image entreaty to?", "What telephone to action would brace good pinch this image?", "What trading transmission would this image beryllium astir effective on?" ] # Get answers marketing_insights = {} for mobility successful marketing_questions: reply = visual_qa(image, question) cardinal = question.split("?")[0].strip().lower().replace(" ", "_") marketing_insights[key] = answer # Visualize nan analysis plt.figure(figsize=(14, 10)) # Display image plt.subplot(1, 2, 1) plt.imshow(np.array(image)) plt.title("Marketing Visual", fontsize=14) plt.axis('off') # Display trading insights plt.subplot(1, 2, 2) plt.axis('off') insights_text = "MARKETING CONTENT ANALYSISnn" for question, cardinal successful zip(marketing_questions, marketing_insights.keys()): insights_text += f"{question}n{marketing_insights[key]}nn" plt.text(0.01, 0.99, insights_text, transform=plt.gca().transAxes, fontsize=12, verticalalignment='top') plt.tight_layout() plt.show() return marketing_insights # Function for societal media understanding def analyze_social_media_content(image_path): # Load image if isinstance(image_path, str): if image_path.startswith(('http://', 'https://')): image = download_image(image_path) else: image = Image.open(image_path).convert('RGB') else: image = image_path # Generate caption caption = generate_caption(image) # Social media circumstantial analysis engagement_potential = visual_qa(image, "How apt is this image to prosecute viewers connected societal media?") suggested_hashtags = visual_qa(image, "What hashtags would beryllium due for this image connected societal media?") platform_fit = visual_qa(image, "Which societal media level would this image execute champion on?") content_type = visual_qa(image, "What type of societal media station would this image beryllium suitable for?") # Create study dict social_analysis = { "caption": caption, "engagement_potential": engagement_potential, "suggested_hashtags": suggested_hashtags, "platform_fit": platform_fit, "content_type": content_type } # Visualize nan analysis plt.figure(figsize=(14, 10)) # Display image plt.subplot(1, 2, 1) plt.imshow(np.array(image)) plt.title("Social Media Content", fontsize=14) plt.axis('off') # Display societal media insights plt.subplot(1, 2, 2) plt.axis('off') insights_text = "SOCIAL MEDIA CONTENT ANALYSISnn" insights_text += f"Caption: {social_analysis['caption']}nn" insights_text += f"Engagement Potential: {social_analysis['engagement_potential']}nn" insights_text += f"Suggested Hashtags: {social_analysis['suggested_hashtags']}nn" insights_text += f"Best Platform: {social_analysis['platform_fit']}nn" insights_text += f"Content Type: {social_analysis['content_type']}n" plt.text(0.01, 0.99, insights_text, transform=plt.gca().transAxes, fontsize=12, verticalalignment='top') plt.tight_layout() plt.show() return social_analysis # Example usage if __name__ == "__main__": # Example: E-commerce merchandise analysis product_url = "https://images.unsplash.com/photo-1598033129183-c4f50c736f10?w=800" print("1. Basic Product Analysis") caption, qa_results = visualize_product_analysis(product_url) print("n2. Creating Automated Product Listing") product_listing = create_product_listing(product_url) print("n3. Marketing Content Analysis") marketing_url = "https://images.unsplash.com/photo-1581252584837-9f0b1d3bf82c?ixlib=rb-4.0.3&q=80" marketing_insights = analyze_marketing_content(marketing_url) print("n4. Social Media Content Analysis") social_url = "https://images.unsplash.com/photo-1534442072653-dbbf80c5e1ae?ixlib=rb-4.0.3&q=80" social_analysis = analyze_social_media_content(social_url)