|
| 1 | +# == csv_to_google_sheet.py Author: Zuinige Rijder ===================================== |
| 2 | +""" |
| 3 | +Simple Python3 script to update google sheet with |
| 4 | +csv input files generated by smart_plug_mini.py |
| 5 | +""" |
| 6 | +import configparser |
| 7 | +import os |
| 8 | +import traceback |
| 9 | +import time |
| 10 | +from datetime import datetime |
| 11 | +from pathlib import Path |
| 12 | +from typing import Generator |
| 13 | +import gspread |
| 14 | + |
| 15 | + |
| 16 | +def log(msg: str) -> None: |
| 17 | + """log a message prefixed with a date/time format yyyymmdd hh:mm:ss""" |
| 18 | + print(datetime.now().strftime("%Y%m%d %H:%M:%S") + ": " + msg) |
| 19 | + |
| 20 | + |
| 21 | +def read_reverse_order(file_name: str) -> Generator[str, None, None]: |
| 22 | + """read in reverse order""" |
| 23 | + # Open file for reading in binary mode |
| 24 | + with open(file_name, "rb") as read_obj: |
| 25 | + # Move the cursor to the end of the file |
| 26 | + read_obj.seek(0, os.SEEK_END) |
| 27 | + # Get the current position of pointer i.e eof |
| 28 | + pointer_location = read_obj.tell() |
| 29 | + # Create a buffer to keep the last read line |
| 30 | + buffer = bytearray() |
| 31 | + # Loop till pointer reaches the top of the file |
| 32 | + while pointer_location >= 0: |
| 33 | + # Move the file pointer to the location pointed by pointer_location |
| 34 | + read_obj.seek(pointer_location) |
| 35 | + # Shift pointer location by -1 |
| 36 | + pointer_location = pointer_location - 1 |
| 37 | + # read that byte / character |
| 38 | + new_byte = read_obj.read(1) |
| 39 | + # If the read byte is newline character then one line is read |
| 40 | + if new_byte == b"\n": |
| 41 | + # Fetch the line from buffer and yield it |
| 42 | + yield buffer.decode()[::-1] |
| 43 | + # Reinitialize the byte array to save next line |
| 44 | + buffer = bytearray() |
| 45 | + else: |
| 46 | + # If last read character is not eol then add it in buffer |
| 47 | + buffer.extend(new_byte) |
| 48 | + # If there is still data in buffer, then it is first line. |
| 49 | + if len(buffer) > 0: |
| 50 | + # Yield the first line too |
| 51 | + yield buffer.decode()[::-1] |
| 52 | + |
| 53 | + |
| 54 | +def get_last_line(filename: Path) -> str: |
| 55 | + """get last line of filename""" |
| 56 | + last_line = "" |
| 57 | + if filename.is_file(): |
| 58 | + with open(filename.name, "rb") as file: |
| 59 | + try: |
| 60 | + file.seek(-2, os.SEEK_END) |
| 61 | + while file.read(1) != b"\n": |
| 62 | + file.seek(-2, os.SEEK_CUR) |
| 63 | + except OSError: |
| 64 | + file.seek(0) |
| 65 | + last_line = file.readline().decode().strip() |
| 66 | + print(f"# last line {filename.name}: {last_line}") |
| 67 | + return last_line |
| 68 | + |
| 69 | + |
| 70 | +def read_csv_and_write_to_sheet( |
| 71 | + array: list, name: str, period: str, params: tuple[str, int, int, int, int, int] |
| 72 | +) -> list: |
| 73 | + """read_csv_and_write_to_sheet""" |
| 74 | + last_line = params[0].split(",") |
| 75 | + last_line_index = params[1] |
| 76 | + startrow = params[2] |
| 77 | + endrow = startrow + params[3] |
| 78 | + strip_begin = params[4] |
| 79 | + strip_end = params[5] |
| 80 | + |
| 81 | + row = startrow |
| 82 | + array.append({"range": f"A{row}:B{row}", "values": [[period, "kWh"]]}) |
| 83 | + row += 1 |
| 84 | + if len(last_line) == 7: |
| 85 | + date_str = last_line[0].strip()[strip_begin:strip_end] |
| 86 | + kwh = float(last_line[last_line_index].strip()) |
| 87 | + array.append({"range": f"A{row}:B{row}", "values": [[date_str, kwh]]}) |
| 88 | + row += 1 |
| 89 | + if period == "Hour": |
| 90 | + csv_postfix = "" |
| 91 | + else: |
| 92 | + csv_postfix = f".{period.lower()}s" |
| 93 | + csv_filename = Path(f"{name}{csv_postfix}.csv") |
| 94 | + if csv_filename.is_file(): |
| 95 | + log(f"### read_reverse_csv_file: {csv_filename.name} " + "#" * 30) |
| 96 | + for line in read_reverse_order(csv_filename.name): |
| 97 | + line = line.strip() |
| 98 | + if line == "": |
| 99 | + continue |
| 100 | + print(line) |
| 101 | + splitted = line.split(",") |
| 102 | + if len(splitted) > 2 and splitted[0].startswith("20"): |
| 103 | + date_str = splitted[0].strip()[strip_begin:strip_end] |
| 104 | + kwh = float(splitted[2].strip()) |
| 105 | + array.append({"range": f"A{row}:B{row}", "values": [[date_str, kwh]]}) |
| 106 | + row += 1 |
| 107 | + |
| 108 | + if row > endrow: |
| 109 | + break # finished |
| 110 | + else: |
| 111 | + log(f"Warning: csv file {csv_filename.name} does not exist") |
| 112 | + |
| 113 | + return array |
| 114 | + |
| 115 | + |
| 116 | +def write_to_sheet(name: str, sheet: list) -> None: |
| 117 | + """write_to_sheet""" |
| 118 | + array = [] |
| 119 | + row = 1 |
| 120 | + last = get_last_line(Path(f"{name}.csv")) |
| 121 | + last_split = last.split(",") |
| 122 | + if len(last_split) != 7 or not last_split[0].strip().startswith("20"): |
| 123 | + log(f"ERROR: unexpected last line in {name}.csv: {last}") |
| 124 | + return |
| 125 | + header = [["Date", "Time", "kWh", "Hour", "Day", "Week", "Month", "Year"]] |
| 126 | + array.append({"range": f"A{row}:H{row}", "values": header}) |
| 127 | + row += 1 |
| 128 | + first_line = [ |
| 129 | + [ |
| 130 | + last_split[0].strip().split(" ")[0].strip()[2:], |
| 131 | + last_split[0].strip().split(" ")[1].strip(), |
| 132 | + float(last_split[1].strip()), |
| 133 | + float(last_split[2].strip()), |
| 134 | + float(last_split[3].strip()), |
| 135 | + float(last_split[4].strip()), |
| 136 | + float(last_split[5].strip()), |
| 137 | + float(last_split[6].strip()), |
| 138 | + ] |
| 139 | + ] |
| 140 | + array.append({"range": f"A{row}:H{row}", "values": first_line}) |
| 141 | + |
| 142 | + # params(last_line, last_line_index, start, rows, strip_start, strip_end) |
| 143 | + array = read_csv_and_write_to_sheet(array, name, "Hour", ("", 2, 3, 49, 8, 16)) |
| 144 | + array = read_csv_and_write_to_sheet(array, name, "Day", (last, 3, 54, 33, 5, 10)) |
| 145 | + array = read_csv_and_write_to_sheet(array, name, "Week", (last, 4, 89, 28, 5, 10)) |
| 146 | + array = read_csv_and_write_to_sheet(array, name, "Month", (last, 5, 119, 26, 0, 7)) |
| 147 | + array = read_csv_and_write_to_sheet(array, name, "Year", (last, 6, 147, 25, 0, 4)) |
| 148 | + |
| 149 | + sheet.batch_update(array) |
| 150 | + |
| 151 | + |
| 152 | +def main() -> None: |
| 153 | + """main""" |
| 154 | + parser = configparser.ConfigParser() |
| 155 | + parser.read("smart_plug_mini.cfg") |
| 156 | + sp3s_settings = dict(parser.items("smart_plug_mini")) |
| 157 | + |
| 158 | + client = gspread.service_account() |
| 159 | + for name in sp3s_settings["device_names"].split(","): |
| 160 | + name = name.strip() |
| 161 | + retries = 2 |
| 162 | + while retries > 0: |
| 163 | + try: |
| 164 | + spreadsheet_name = f"{name}.SP" |
| 165 | + log(f"##### Writing {spreadsheet_name} " + "#" * 60) |
| 166 | + spreadsheet = client.open(spreadsheet_name) |
| 167 | + sheet = spreadsheet.sheet1 |
| 168 | + sheet.clear() |
| 169 | + write_to_sheet(name, sheet) |
| 170 | + |
| 171 | + retries = 0 |
| 172 | + except Exception as ex: # pylint: disable=broad-except |
| 173 | + log("Exception: " + str(ex)) |
| 174 | + traceback.print_exc() |
| 175 | + retries -= 1 |
| 176 | + log("Sleeping a minute") |
| 177 | + time.sleep(60) |
| 178 | + |
| 179 | + |
| 180 | +main() |
0 commit comments