opendatade / html2csv Goto Github PK

...
#TODO: In HTML documents that use tables for layout (yuck!) add option to detect table titles in preceding sibling tables of size 1x1
...

...
    #Removes nested tables. for handling the sins of 1990's web pages.
    #TODO: Add an argument to enable/disable table de-nesting
    [t.extract() for t in table.findAll("table")]
    #This would grab all TRs regardless of depth without the above line removing nested tables
    for row in table.findAll('tr'):
...

...
#TODO: Add a -v/--verbose option for the excessive print statements below. Consider both `-v` `-V` (AKA: VERY VERBOSE, which is currently the default)
...
    for row in table.findAll('tr'):
      print(f"Processing row number {rowcount}")
      rowcount += 1
      cols = row.findAll(['td','th'])
      print(f"Found {len(cols)} columns.")
...

...
#TODO: Add option to compress multi-line categorical headers (eg: headers with colspan>1) into concatenated-naming-style single row headers
#  EG: "Population" header spanning above sub-headers "Number" and "Percentage" produces two single-column headers of "Population - Number"
#  and "Population - Percentage". Delemiter can be configurable with a sensible default set.
...

...
      print(f"Found {len(cols)} columns.")
      #TODO: Detect and discard empty tables (those that contain 1 row, 1 column, consisting of an empty string...I think?)
      #TODO: Add a warning for non-rectangular tables.
      if cols:
        cols = [str(x.text).strip() for x in cols]
...

...
      cols = row.findAll(['td','th'])
      print(f"Found {len(cols)} columns.")
      #TODO: Detect and discard empty tables (those that contain 1 row, 1 column, consisting of an empty string...I think?)
      #TODO: Add a warning for non-rectangular tables.
...