Correctly remove duplicates in site summaries

waternsw_grabber/waternsw_grabber.py

@@ -379,19 +379,22 @@ def extract_definitions(input_dir, output_dir):
 
         try:
             sites = sites.append(pd.read_csv(csv_name_s, index_col=0))
-            sites = sites.drop_duplicates().sort_index()
+            sites = sites[~sites.index.duplicated(keep='first')]
+            sites = sites.sort_index()
         except FileNotFoundError:
             pass
 
         try:
             variables = variables.append(pd.read_csv(csv_name_v, index_col=0))
-            variables = variables.drop_duplicates().sort_index()
+            variables = variables[~variables.index.duplicated(keep='first')]
+            variables = variables.sort_index()
         except FileNotFoundError:
             pass
 
         try:
-            variables = variables.append(pd.read_csv(csv_name_q, index_col=0))
-            qualities = qualities.drop_duplicates().sort_index()
+            qualities = qualities.append(pd.read_csv(csv_name_q, index_col=0))
+            qualities = qualities[~qualities.index.duplicated(keep='first')]
+            qualities = qualities.sort_index()
         except FileNotFoundError:
             pass