device = "cuda" if torch.cuda.is_available() else "cpu" model, preprocess = clip.load("vit-b/32", device = device) 

def extract_frames(video_path):     frames = []     cap = cv2.videocapture(video_path)     frame_rate = cap.get(cv2.cap_prop_fps)     total_frames = int(cap.get(cv2.cap_prop_frame_count))     total_seconds = total_frames / frame_rate     target_frame_count = int(total_seconds)     target_frame_index = 0     for i in range(target_frame_count):         cap.set(cv2.cap_prop_pos_frames, target_frame_index)         ret, frame = cap.read()         if not ret:             break         frames.append(frame)         target_frame_index += int(frame_rate)     cap.release()     return frames 

def generate_embedding(frame):     frame_tensor = preprocess(pilimage.fromarray(frame)).unsqueeze(0).to(device)     with torch.no_grad():         embedding = model.encode_image(frame_tensor).cpu().numpy()     return embedding[0] 

def store_frame_embedding_and_image(video_path):     frames = extract_frames(video_path)     data = [         (i+1, generate_embedding(frame), frame)         for i, frame in enumerate(tqdm(             frames,             desc = "processing frames",             bar_format = "{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}    <p>让我们检查一下 dataframe 中存储的数据的大小特征：<br></p>  <pre class="brush:php;toolbar:false">embedding_lengths = df["embedding_data"].str.len() frame_lengths = df["frame_data"].str.len()  # calculate min and max lengths for embeddings and frames min_embedding_length, max_embedding_length = embedding_lengths.min(), embedding_lengths.max() min_frame_length, max_frame_length = frame_lengths.min(), frame_lengths.max()  # print results print(f"min length of embedding vectors: {min_embedding_length}") print(f"max length of embedding vectors: {max_embedding_length}") print(f"min length of frame data vectors: {min_frame_length}") print(f"max length of frame data vectors: {max_frame_length}") 

min length of embedding vectors: 512 max length of embedding vectors: 512 min length of frame data vectors: 1080 max length of frame data vectors: 1080 

def calculate_similarity(query_embedding, df):     # convert the query embedding to a tensor     query_tensor = torch.tensor(query_embedding, dtype = torch.float32).to(device)      # convert the list of embeddings to a numpy array     embeddings_np = np.array(df["embedding_data"].tolist())      # create a tensor from the numpy array     embeddings_tensor = torch.tensor(embeddings_np, dtype = torch.float32).to(device)      # compute similarities using matrix multiplication     similarities = torch.mm(embeddings_tensor, query_tensor.unsqueeze(1)).squeeze().tolist()     return similarities 

def encode_text_query(query):     # tokenize the query text     tokens = clip.tokenize([query]).to(device)      # compute text features using the pretrained model     with torch.no_grad():         text_features = model.encode_text(tokens)      # convert the tensor to a numpy array and return it     return text_features.cpu().numpy().flatten() 

query = input("enter your query: ") text_query_embedding = encode_text_query(query) text_similarities = calculate_similarity(text_query_embedding, df) df["text_similarity"] = text_similarities 

# retrieve the top 5 text matches based on similarity top_text_matches = df.nlargest(5, "text_similarity")  print("top 5 best matches:") print(top_text_matches[["frame_number", "text_similarity"]].to_string(index = false)) 

top 5 best matches:  frame_number  text_similarity            40        36.456184            39        36.081161            43        33.295975            42        32.423229            45        31.931164 

def plot_frames(frames, frame_numbers):     num_frames = len(frames)     fig, axes = plt.subplots(1, num_frames, figsize = (15, 5))      for ax, frame_data, frame_number in zip(axes, frames, frame_numbers):         ax.imshow(frame_data)         ax.set_title(f"frame {frame_number}")         ax.axis("off")      plt.tight_layout()     plt.show()  # collect frame data and numbers for the top text matches top_text_matches_indices = top_text_matches.index.tolist() frames = [df.at[index, "frame_data"] for index in top_text_matches_indices] frame_numbers = [df.at[index, "frame_number"] for index in top_text_matches_indices]  # plot the frames plot_frames(frames, frame_numbers) 

def encode_image_query(image):     # preprocess the image and add batch dimension     image_tensor = preprocess(image).unsqueeze(0).to(device)      # extract features using the model     with torch.no_grad():         image_features = model.encode_image(image_tensor)      # convert features to numpy array and flatten     return image_features.cpu().numpy().flatten() 

image_url = "https://github.com/veryfatboy/clip-demo/raw/main/thumbnails/1_what_makes_singlestore_unique.png"  response = requests.get(image_url)  if response.status_code == 200:     display(image(url = image_url))     image_file = pilimage.open(bytesio(response.content))      image_query_embedding = encode_image_query(image_file)     image_similarities = calculate_similarity(image_query_embedding, df)     df["image_similarity"] = image_similarities else:     print("failed to download the image, status code:", response.status_code) 

top_image_matches = df.nlargest(5, "image_similarity")  print("top 5 best matches:") print(top_image_matches[["frame_number", "image_similarity"]].to_string(index = false)) 

top 5 best matches:  frame_number  image_similarity             7         57.674603             9         43.669739             6         42.573799            15         40.296551            93         40.201733 

# collect frame data and numbers for the top image matches top_image_matches_indices = top_image_matches.index.tolist() frames = [df.at[index, "frame_data"] for index in top_image_matches_indices] frame_numbers = [df.at[index, "frame_number"] for index in top_image_matches_indices]  # plot the frames plot_frames(frames, frame_numbers) 

combined_query_embedding = (text_query_embedding + image_query_embedding) / 2 combined_similarities = calculate_similarity(combined_query_embedding, df) df["combined_similarity"] = combined_similarities 

top_combined_matches = df.nlargest(5, "combined_similarity")  print("top 5 best matches:") print(top_combined_matches[["frame_number", "combined_similarity"]].to_string(index = false)) 

top 5 best matches:  frame_number  combined_similarity             7            36.337120             5            32.869991             6            32.559093            93            32.205418            94            31.881357 

# collect frame data and numbers for the top combined matches top_combined_matches_indices = top_combined_matches.index.tolist() frames = [df.at[index, "frame_data"] for index in top_combined_matches_indices] frame_numbers = [df.at[index, "frame_number"] for index in top_combined_matches_indices]  # plot the frames plot_frames(frames, frame_numbers) 

frames_df = df.copy() frames_df.drop(     columns = ["text_similarity", "image_similarity", "combined_similarity"],     inplace = true )  query_string = combined_query_embedding.copy() 

def process_data(arr):     return np.array2string(arr, separator = ",").replace("n", "")  frames_df["embedding_data"] = frames_df["embedding_data"].apply(process_data) frames_df["frame_data"] = frames_df["frame_data"].apply(process_data) query_string = process_data(query_string) 

shared_tier_check = %sql show variables like "is_shared_tier" if not shared_tier_check or shared_tier_check[0][1] == "off":     %sql drop database if exists video_db;     %sql create database if not exists video_db; 

from sqlalchemy import *  db_connection = create_engine(connection_url) 

drop table if exists frames;  create table if not exists frames (     frame_number int(10) unsigned not null,     embedding_data vector(512) not null,     frame_data text,     key(frame_number) ); 

frames_df.to_sql(     "frames",     con = db_connection,     if_exists = "append",     index = false,     chunksize = 1000 ) 

select frame_number,     substring(embedding_data, 1, 50) as embedding_data,     substring(frame_data, 1, 50) as frame_data from frames limit 1; 

alter table frames add vector index (embedding_data)      index_options '{           "index_type":"auto",           "metric_type":"dot_product"      }'; 

select frame_number,     embedding_data  :query_string as similarity from frames order by similarity use index () desc limit 5; 

select frame_number,     embedding_data  :query_string as similarity from frames order by similarity desc limit 5; 

sql_query = """ SELECT frame_number, embedding_data, frame_data FROM frames ORDER BY embedding_data  %s DESC LIMIT 5; """  new_frames_df = pd.read_sql(     sql_query,     con = db_connection,     params = (query_string,) )  new_frames_df.head()