check_for_CID.py

import csv
import os

def remove_duplicates_from_csv(input_file, output_file):
   
    total_rows = 0
    
    with open(input_file, 'r', newline='') as infile, open(output_file, 'w', newline='') as outfile:
        reader = csv.reader(infile)
        writer = csv.writer(outfile)
        
        # Skip the header row
        next(reader)
        
        for row in reader:
            cleaned_row = list(set(row))
            writer.writerow(cleaned_row)
            total_rows += 1
            
    return total_rows

if __name__ == "__main__":
    # input_csv_file = r"C:\Users\user\Desktop\COSMOSAC-master\COSMO\COSMO_all\s-profiles-all\check\check.csv"
    input_csv_file = r"C:\Users\user\Desktop\COSMOSAC-master\COSMO\COSMO_all\s-profiles-all\check\check_2.csv"
    output_csv_file = r"C:\Users\user\Desktop\COSMOSAC-master\COSMO\COSMO_all\s-profiles-all\check\gjf_checked_2.csv"
    # output_csv_file = r"C:\Users\user\Desktop\COSMOSAC-master\COSMO\COSMO_all\s-profiles-all\check\name_checked_2.csv"
    total_rows = remove_duplicates_from_csv(input_csv_file, output_csv_file)
    print(f"Duplicates removed from {input_csv_file}. Cleaned data saved to {output_csv_file}.")
    print(f"Total number of rows in cleaned CSV (excluding header): {total_rows}")