GeezerVisonThe World Wide Web, restored to 800 pixels of glory. |
| Home|About|Gaming cabinet|Archives|How it works|Starter sites |
GeezerVison is showing github.com — python-deadlines/utils/import_python_organizers.py at main · python-spain/python-deadlines Expand file tree/ Copy pathimport_python_organizers.pyMore file actions 420 lines (349 loc) · 15.5 KB / Copy pathimport_python_organizers.pyFile metadata and controls420 lines (349 loc) · 15.5 KB 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 # Standard library from datetime import datetime from datetime import timezone from pathlib import Path from urllib import error as urllib_error # Third-party import pandas as pd # Local imports try: from tidy_conf import fuzzy_match from tidy_conf import load_conferences from tidy_conf import merge_conferences from tidy_conf.countries import get_country_alpha3 from tidy_conf.deduplicate import deduplicate from tidy_conf.schema import get_schema from tidy_conf.titles import normalize_conference_name from tidy_conf.utils import fill_missing_required from tidy_conf.yaml import load_title_mappings from tidy_conf.yaml import write_df_yaml except ImportError: from .tidy_conf import fuzzy_match from .tidy_conf import load_conferences from .tidy_conf import merge_conferences from .tidy_conf.countries import get_country_alpha3 from .tidy_conf.deduplicate import deduplicate from .tidy_conf.schema import get_schema from .tidy_conf.titles import normalize_conference_name from .tidy_conf.utils import fill_missing_required from .tidy_conf.yaml import load_title_mappings from .tidy_conf.yaml import write_df_yaml def load_remote(year: int) -> pd.DataFrame: """Load conference data from GitHub CSV for a specific year. Parameters ---------- year : int The year to load conference data for Returns ------- pd.DataFrame DataFrame containing conference data from the CSV """ url = f"https://raw.githubusercontent.com/python-organizers/conferences/main/{year}.csv" # Read data and rename columns df = pd.read_csv(url) df = map_columns(df) # Only return valid cfps # return df.dropna(subset=['cfp']) return df def map_columns(df: pd.DataFrame, reverse: bool = False) -> pd.DataFrame: """Map columns between CSV format and conference schema. Parameters ---------- df : pd.DataFrame DataFrame with columns to map reverse : bool, optional If True, map from schema to CSV format. Default is False Returns ------- pd.DataFrame DataFrame with mapped columns """ cols = { "Subject": "conference", "Start Date": "start", "End Date": "end", "Tutorial Deadline": "tutorial_deadline", "Talk Deadline": "cfp", "Website URL": "link", "Proposal URL": "cfp_link", "Sponsorship URL": "sponsor", } df["place"] = df["Location"] if reverse: cols = {v: k for k, v in cols.items()} return df.rename(columns=cols) def write_csv(df: pd.DataFrame, year: int, csv_location: str) -> None: """Write the CSV files for the conferences. Parameters ---------- df : pd.DataFrame DataFrame containing conference data to write year : int The year for the CSV file csv_location : str Directory path where CSV files should be written """ from logging_config import get_tqdm_logger logger = get_tqdm_logger(__name__) logger.info(f"Starting write_csv for year {year} with df shape: {df.shape}") logger.debug(f"write_csv input columns: {df.columns.tolist()}") # Validate and fix conference names before processing invalid_mask = ~df["conference"].apply( lambda x: isinstance(x, str) and len(str(x).strip()) > 0, ) invalid_conferences = df[invalid_mask] if not invalid_conferences.empty: logger.error( f"Found {len(invalid_conferences)} rows with invalid conference names in write_csv:", ) for idx, row in invalid_conferences.iterrows(): logger.error( f" Row {idx}: conference = {row['conference']} (type: {type(row['conference'])})", ) # Fix invalid conference names with proper indexing for idx in invalid_conferences.index: original_value = df.at[idx, "conference"] if pd.notna(original_value) and str(original_value).strip(): df.at[idx, "conference"] = str(original_value).strip() else: df.at[idx, "conference"] = f"Conference_{idx}" # Sanitize CFP and deadline data safely df["cfp"] = df["cfp"].fillna("").astype(str).str.slice(stop=10).str.replace(r"\b(TBA|None)\b", "", regex=True) df["tutorial_deadline"] = ( df["tutorial_deadline"].fillna("").astype(str).str.slice(stop=10).str.replace(r"\b(TBA|None)\b", "", regex=True) ) # Ensure empty strings instead of nan values df["cfp"] = df["cfp"].replace("nan", "") df["tutorial_deadline"] = df["tutorial_deadline"].replace("nan", "") # Map columns back to CSV format df = map_columns(df, reverse=True) # Additional cleaning after column mapping to ensure consistency if "Talk Deadline" in df.columns: df["Talk Deadline"] = df["Talk Deadline"].fillna("").astype(str).replace("nan", "") if "Tutorial Deadline" in df.columns: df["Tutorial Deadline"] = df["Tutorial Deadline"].fillna("").astype(str).replace("nan", "") logger.debug(f"After map_columns, df shape: {df.shape}") for y in range(year, datetime.now(tz=timezone.utc).year + 10): # Extract and prepare data for this year (even if empty) df_year_subset = df.loc[df["year"] == y] if y in df["year"].unique() else pd.DataFrame(columns=df.columns) logger.debug(f"Year {y} subset shape: {df_year_subset.shape}") # Only create CSV if we have data or if the original df was not empty (to handle empty year subsets) if not df_year_subset.empty or df.empty: csv_data = ( df_year_subset[ [ "Subject", "Start Date", "End Date", "Location", "Country", "Venue", "Tutorial Deadline", "Talk Deadline", "Website URL", "Proposal URL", "Sponsorship URL", ] ] .fillna("") .astype(str) .replace("nan", "") # Convert string "nan" back to empty string .sort_values(by=["Start Date", "End Date", "Subject"]) if not df_year_subset.empty else df_year_subset[ [ "Subject", "Start Date", "End Date", "Location", "Country", "Venue", "Tutorial Deadline", "Talk Deadline", "Website URL", "Proposal URL", "Sponsorship URL", ] ] .fillna("") .astype(str) ) logger.debug(f"Writing CSV for year {y} with {len(csv_data)} conferences") if not csv_data.empty: logger.debug( f"Sample conference names: {csv_data['Subject'].head().tolist()}", ) if "Talk Deadline" in csv_data.columns: logger.debug( f"Talk Deadline values before CSV write: {csv_data['Talk Deadline'].tolist()}", ) csv_data.to_csv(Path(csv_location, f"{y}.csv"), index=False) logger.info(f"Successfully wrote {Path(csv_location, f'{y}.csv')}") def main(year: int | None = None, base: str = "") -> None: """Import Python conferences from a csv file on Github. Parameters ---------- year : int | None, optional Starting year for import. If None, uses current year base : str, optional Base directory path for data files. Default is empty string """ from logging_config import get_tqdm_logger # Setup tqdm-compatible logging for this module logger = get_tqdm_logger(__name__) logger.info("🚀 Starting import_python_organizers main function") # If no year is provided, use the current year if year is None: year = datetime.now(tz=timezone.utc).year logger.info(f"Processing conferences for year: {year}") # Load current conferences data_path = Path(base, "_data") utils_path = Path(base, "utils") tmp_path = Path(base, ".tmp") tmp_path.mkdir(exist_ok=True, parents=True) data_path.mkdir(exist_ok=True, parents=True) target_file = Path(data_path, "conferences.yml") csv_location = Path(utils_path, "conferences") cache_file = Path(tmp_path, ".conferences_py_orgs.csv") # Load the existing conference data df_yml = load_conferences() df_schema = get_schema() df_new = pd.DataFrame(columns=df_schema.columns) df_csv_raw = pd.DataFrame(columns=df_schema.columns) # Parse your csv file and iterate through year by year for y in range(year, datetime.now(tz=timezone.utc).year + 10): try: df = deduplicate(load_remote(year=y), "conference") df["year"] = y except urllib_error.HTTPError: break df_csv_raw = pd.concat([df_csv_raw, df], ignore_index=True) # Load old csv dataframe from cached data # try: # df_csv_old = pd.read_csv(cache_file) # except FileNotFoundError: # df_csv_old = pd.DataFrame(columns=df_csv_raw.columns) # Create a copy for processing with standardized names df_csv_standardized = df_csv_raw.copy() # Load and apply the title mappings _, known_mappings = load_title_mappings(reverse=True) # CRITICAL: Use normalize_conference_name for CONSISTENT normalization # This ensures the same normalization is used here AND in mapping_dict later df_csv_standardized["conference"] = df_csv_standardized["conference"].apply( lambda x: normalize_conference_name(x, known_mappings), ) # Store the new csv dataframe to cache (with original names) df_cache = df_csv_raw.copy() # Get the difference between the old and new dataframes # _ = pd.concat([df_csv_old, df_csv_raw]).drop_duplicates(keep=False) # Deduplicate the new dataframe (with standardized names for merging) # CRITICAL: Must group by both conference AND year to avoid losing multi-year entries # (e.g., "PyCon USA 2025" and "PyCon USA 2026" both normalize to "PyCon USA") df_csv_for_merge = deduplicate(df_csv_standardized, ["conference", "year"]) if df_csv_for_merge.empty: print("No new conferences found in Python organiser source.") return # Process year by year for y in range(year, datetime.now(tz=timezone.utc).year + 10): if df_csv_for_merge.loc[df_csv_for_merge["year"] == y].empty or df_yml[df_yml["year"] == y].empty: # Concatenate the new data with the existing data df_new = pd.concat( [ df_new, df_yml[df_yml["year"] == y], df_csv_for_merge.loc[df_csv_for_merge["year"] == y], ], ignore_index=True, ) continue logger.info(f"Processing year {y} merge operations") df_yml_year = df_yml[df_yml["year"] == y] df_csv_year = df_csv_for_merge.loc[df_csv_for_merge["year"] == y] logger.debug( f"Year {y}: df_yml_year shape: {df_yml_year.shape}, df_csv_year shape: {df_csv_year.shape}", ) df_merged, df_remote, merge_report = fuzzy_match(df_yml_year, df_csv_year) logger.info( f"Merge report: {merge_report.exact_matches} exact, " f"{merge_report.fuzzy_matches} fuzzy, {merge_report.no_matches} no match", ) logger.info( f"Fuzzy match completed for year {y}. df_merged shape: {df_merged.shape}", ) df_merged["year"] = y df_merged = df_merged.drop(["conference"], axis=1) logger.debug(f"After dropping conference column: {df_merged.shape}") df_merged = deduplicate(df_merged) df_remote = deduplicate(df_remote) logger.debug( f"After deduplication - df_merged: {df_merged.shape}, df_remote: {df_remote.shape}", ) df_merged = merge_conferences(df_merged, df_remote) logger.info( f"Merge conferences completed for year {y}. Final shape: {df_merged.shape}", ) df_new = pd.concat([df_new, df_merged], ignore_index=True) # Fill in missing required fields df_new = fill_missing_required(df_new) # Write the new data to the YAML file write_df_yaml(df_new, target_file) # Prepare CSV output with original names df_csv_output = df_csv_raw.copy() # Map from the standardized data back to original # CRITICAL: Use the SAME normalization function as tidy_df_names to avoid data loss mapping_dict = {} for idx, row in df_csv_raw.iterrows(): # Use normalize_conference_name for consistent normalization standardized_conf = normalize_conference_name(row["conference"], known_mappings) mapping_key = (standardized_conf, row["year"]) mapping_dict[mapping_key] = idx # Track entries that matched and those that didn't for debugging matched_keys = set() unmatched_entries = [] # Update the CSV output with information from the merged data for _, row in df_new.iterrows(): key = (row["conference"], row["year"]) if key in mapping_dict: original_idx = mapping_dict[key] matched_keys.add(key) # Update only fields that were potentially enriched during merge for col in ["start", "end", "cfp", "link", "cfp_link", "sponsor", "finaid"]: if col in row and pd.notna(row[col]): df_csv_output.at[original_idx, col] = row[col] else: # Track entries that didn't match for potential debugging unmatched_entries.append( {"conference": row["conference"], "year": row["year"]}, ) # Log any unmatched entries for debugging (these may be legitimately new) if unmatched_entries: logger.debug( f"Found {len(unmatched_entries)} entries in df_new not in mapping_dict " "(may be new conferences from YAML):", ) for entry in unmatched_entries[:5]: # Show first 5 logger.debug(f" - {entry['conference']} ({entry['year']})") # Verify no silent data loss: all CSV entries should be accounted for csv_keys_in_mapping = set(mapping_dict.keys()) unmatched_csv = csv_keys_in_mapping - matched_keys if unmatched_csv: logger.warning( f"Potential data loss: {len(unmatched_csv)} CSV entries were not updated:", ) for key in list(unmatched_csv)[:5]: # Show first 5 logger.warning(f" - {key[0]} ({key[1]})") # Write the CSV with original names df_csv_output.loc[:, "Location"] = df_csv_output.place # Extract country from place (format: "City, Country") and convert to alpha3 code # Uses get_country_alpha3 which preserves original country name if lookup fails df_csv_output.loc[:, "Country"] = df_csv_output.place.str.split(",").str[-1].str.strip().apply(get_country_alpha3) write_csv(df_csv_output, year, csv_location) # Save the new dataframe to cache df_cache.to_csv(cache_file, index=False) if __name__ == "__main__": # Make argparse to get year and base import argparse parser = argparse.ArgumentParser(description="Import Python Organizers") parser.add_argument("--year", type=int, help="Year to import") main(year=parser.parse_args().year) |
| You are visitor 01402520 — GeezerVison does not store the pages you look up. Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required. |