AutoComp/concagate.py at master · LBAI-InfoLab/AutoComp · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
##
## Concat CSV file generated by transfogate function
##  - generate one result file : "cyto_merged.csv"
##  - generate one log file : "cyto_merged_drop_omicid.csv"
##
## When no arguments is given or "all" arguments is provided as the first
## argument the programm scan the current directory and add all _transformed.csv
## file to the target list
##
## exemple :
##      > python concagate.py all
##      > python concagate.py file1.csv file2.csv file3.csv
##


def concagate(target_list):
    """
    concat all csv file in target_list, write a log file with the
    missing omicid and the file where the were not found.

        -> target_list is a list of file name
        -> Write a results files
        -> target file are supposed to be generated with the transfogate
           function
    """

    ## importation
    import pandas as pd
    import glob

    ## parameters
    main_dataframe = pd.DataFrame()
    output_data_file_name = "cyto_merged.csv"
    drop_file_name = "cyto_merged_drop_omicid.csv"
    all_patients = []
    patients_drop = []

    ## init main dataset
    main_dataframe = pd.read_csv(target_list[0], sep=";")

    ## get the patients id
    all_patients += list(main_dataframe["patient ID"])

    ## add iteratively the fetched datasets
    for target_file in target_list[1:]:

        ## load dataset
        dataset = pd.read_csv(target_file, sep=";")

        ## get the patients id
        for patient_id in list(dataset["patient ID"]):
            if(patient_id not in all_patients):
                all_patients.append(patient_id)

        ## drop center variable to avoid duplicate, the other
        ## descriptive variables are supposed to be unique at least
        ## in panel 1, 3 and 5.
        dataset = dataset.drop(columns =['center'])

        ## merge datasets
        main_dataframe = pd.merge(main_dataframe, dataset, on='patient ID')

    ## Write final dataset
    main_dataframe.to_csv(output_data_file_name, index=False)


    ## create drop list patients
    log_file = open(drop_file_name, "w")
    for patient_id in all_patients:
        if(patient_id not in list(main_dataframe["patient ID"])):

            ## init missing values list
            missing_values = []

            ## add id in drop list
            patients_drop.append(patient_id)

            ## locate missing values
            for dataset_file in target_list:
                dataset = pd.read_csv(dataset_file, sep=";")
                if(patient_id not in list(dataset['patient ID'])):
                    missing_values.append(dataset_file)

            ## Display Missing search
            print("[INFO] NaN values for "+str(patient_id) +" in files : "+str(missing_values))

            ## write entry in the log file
            log_file.write(str(patient_id)+", missing values in : "+str(missing_values)+"\n")

    ## close log file
    log_file.close()

    ## display drop count
    print("[INFO] drop "+str(len(patients_drop))+ " patients among "+str(len(all_patients)))


if __name__ == "__main__":
    ## Main module, parse arguments and
    ## run concagate function if a valid target list
    ## of files is found

    ## importation
    import glob
    import sys
    import os.path

    ## parameters
    arguments = sys.argv
    target_list = []

    ## get arguments and define targets
    if(len(arguments) > 1):
        if(arguments[1] == "all"):
            target_list = glob.glob("*_transformed.csv")
        else:
            for elt in arguments:
                if(os.path.isfile(elt)):
                    target_list.append(elt)
                else:
                    print("[WARNING] "+str(elt) +" is not a file")
    else:
        target_list = glob.glob("*_transformed.csv")


    ## Check target list and run concagate if
    ## the list contain at least two files
    if(len(target_list) < 2):
        print("[INFO] unvalid target list")
    else:
        ## run concagate
        concagate(target_list)