From 38579d8c1d340a8c39b874db880d2855f3f4ce5c Mon Sep 17 00:00:00 2001 From: James Miller Date: Mon, 11 Nov 2019 23:42:47 -0600 Subject: [PATCH] got skills working with yaml --- data/yaml/gendb.py | 563 ++++++++++++++++++++++++++---------------- data/yaml/skills.yaml | 87 +++++++ 2 files changed, 439 insertions(+), 211 deletions(-) create mode 100644 data/yaml/skills.yaml diff --git a/data/yaml/gendb.py b/data/yaml/gendb.py index 214b5bd..656e2e8 100644 --- a/data/yaml/gendb.py +++ b/data/yaml/gendb.py @@ -6,38 +6,171 @@ import sys DBFILE = 'tmp.db' + def main(): - # delete DBfile and run fresh - os.remove(DBFILE) - # Load in the yaml data - with open('basics.yaml') as yl: - data = yaml.full_load(yl) - # Get a DB conn - conn = get_db_conn() - # call the functions to input to SQL - do_abilityscore(data['abilityscore'], conn) - do_actioncost(data['actioncost'], conn) - do_alignment(data['alignment'], conn) - do_frequency(data['frequency'], conn) - do_langrarity(data['lang_rarity'], conn) - do_movement(data['movement'], conn) - do_size(data['size'], conn) - do_weaponcategory(data['weaponcategory'], conn) + # delete DBfile and run fresh + os.remove(DBFILE) + # Load in the yaml data + with open('basics.yaml') as yl: + data = yaml.full_load(yl) + # Get a DB conn + conn = get_db_conn() + # call the functions to input to SQL + do_abilityscore(data['abilityscore'], conn) + do_actioncost(data['actioncost'], conn) + do_alignment(data['alignment'], conn) + do_frequency(data['frequency'], conn) + do_langrarity(data['lang_rarity'], conn) + do_movement(data['movement'], conn) + do_size(data['size'], conn) + do_weaponcategory(data['weaponcategory'], conn) - # move on to traits - with open('traits.yaml') as yl: - data = yaml.full_load(yl) - do_traits(data, conn) # does both trait types and traits + # move on to traits + with open('traits.yaml') as yl: + data = yaml.full_load(yl) + do_traits(data, conn) # does both trait types and traits - # move on to sources - with open('sources.yaml') as yl: - data = yaml.full_load(yl) - do_sources(data, conn) + # move on to sources + with open('sources.yaml') as yl: + data = yaml.full_load(yl) + do_sources(data, conn) + do_source_entry_table(conn) + + # move on to skills + with open('skills.yaml') as yl: + data = yaml.full_load(yl) + do_skills(data, conn) + +def do_skills(data, conn): + # make skill table + table = """ +CREATE TABLE skill ( + skill_id INTEGER PRIMARY KEY, + "name" TEXT UNIQUE NOT NULL, + descr TEXT +); + """ + c = conn.cursor() + c.execute(table) + + # make sourceentry many-to-many table + table = """ +CREATE TABLE sourceentry_skill ( + id INTEGER PRIMARY KEY, + sourceentry_id INTEGER NOT NULL, + skill_id INTEGER NOT NULL, + UNIQUE (sourceentry_id, skill_id), -- prevent duplicates + FOREIGN KEY (sourceentry_id) REFERENCES sourceentry(sourceentry_id), + FOREIGN KEY (skill_id) REFERENCES skill(skill_id) +); + """ + c = conn.cursor() + c.execute(table) + + # insert basics into skill table + inp_data = [] + for i in data['skill']: + inp_data.append( + (i['name'], i['descr'])) + + stmt = "INSERT INTO skill (name, descr) VALUES (?,?)" + try: + conn.executemany(stmt, inp_data) + except sqlite3.Error as e: + print("Error creating skill: {}".format(e)) + except: + print("Error creating skill something other than sqlite3 error") + else: + conn.commit() + + # go through and do source entry linking + + for i in data['skill']: + print("\n\nDoing the skill: {}".format(i['name'])) + srcs = [] + # TODO refactor this inner loop for sources out + for j in i['source']: + abbr = j['abbr'] + page_start = j['page_start'] + if 'page_stop' in j: + page_stop = j['page_stop'] + else: + page_stop = page_start + srcs.append([i['name'], abbr, page_start, page_stop]) + print("srcs: {}".format(srcs)) + do_sourceentry_to_skill(srcs, conn) + + +# TODO ugggh;;; this is soooo ugly and needs refactoring but it's working +def do_sourceentry_to_skill(srcs, conn): + c = conn.cursor() + + stmt = "SELECT source.source_id, skill.skill_id FROM source, skill WHERE source.abbr=? AND skill.name=?" + istmt = "INSERT INTO sourceentry (source_id, page_start, page_stop) VALUES (?,?,?)" + for i in srcs: + print("i in srcs: {}".format(i)) + inp_data=(i[1],i[0]) + print("inp data: {}".format(inp_data)) + for row in c.execute(stmt, inp_data): + print("source_id:{} skill_id:{}".format(row[0], row[1])) + iinp_data = (row[0], i[2], i[3]) + print("iinp data: {}".format(iinp_data)) + + try: + c.execute(istmt, iinp_data) + except sqlite3.IntegrityError as e: + if "UNIQUE" in str(e): + # we fully expect UNIQUE constraint to fail on some of these so it's fine + conn.commit() + print("committed istmt") + else: + # but we still want to know what's going on if there's some other error + print("Something went wrong with istmt: {}".format(e)) + except sqlite3.Error as e: + print("Error inserting a sourceentry for skill: {}".format(e)) + else: + conn.commit() + print("committed istmt") + + linkstmt = "INSERT INTO sourceentry_skill (sourceentry_id, skill_id) VALUES ((SELECT sourceentry_id from sourceentry WHERE source_id=? AND page_start=? AND page_stop=?), ?)" + linkinp_data = (row[0], i[2], i[3], row[1]) + print(linkinp_data) + try: + c.execute(linkstmt, linkinp_data) + except sqlite3.IntegrityError as e: + if "UNIQUE" in str(e): + # we fully expect UNIQUE constraint to fail on some of these so it's fine + conn.commit() + print("committed linkstmt") + pass + else: + # but we still want to know what's going on if there's some other error + print(e) + except sqlite3.Error as e: + print("Error inserting a sourceentry for skill: {}".format(e)) + else: + conn.commit() + print("committed linkstmt") + + +def do_source_entry_table(conn): + table = """ +CREATE TABLE sourceentry ( + sourceentry_id INTEGER PRIMARY KEY, + source_id INTEGER NOT NULL, + page_start INTEGER NOT NULL, + page_stop INTEGER NOT NULL, + UNIQUE (source_id, page_start, page_stop), -- prevent duplicates + FOREIGN KEY (source_id) REFERENCES source(source_id) +); + """ + c = conn.cursor() + c.execute(table) def do_sources(data, conn): - table = """ -CREATE TABLE sources ( - sources_id INTEGER PRIMARY KEY, + table = """ +CREATE TABLE source ( + source_id INTEGER PRIMARY KEY, isbn TEXT, pzocode TEXT, full_name TEXT NOT NULL UNIQUE, @@ -50,40 +183,36 @@ CREATE TABLE sources ( ); """ - c = conn.cursor() - c.execute(table) + c = conn.cursor() + c.execute(table) - inp_data = [] - for i in data['source']: - inp_data.append((i['isbn'], - i['pzocode'], - i['full_name'], - i['short_name'], - i['abbr'], - i['descr'], - i['release_date'], - i['is_first_party'], - i['ogl_copyright_block'])) + inp_data = [] + for i in data['source']: + inp_data.append( + (i['isbn'], i['pzocode'], i['full_name'], i['short_name'], + i['abbr'], i['descr'], i['release_date'], i['is_first_party'], + i['ogl_copyright_block'])) + + stmt = "INSERT INTO source (isbn, pzocode, full_name, short_name, abbr, descr, release_date, is_first_party, ogl_copyright_block) VALUES (?,?,?,?,?,?,?,?,?)" + try: + conn.executemany(stmt, inp_data) + except sqlite3.Error as e: + print("Error creating source: {}".format(e)) + except: + print("Error creating sources something other than sqlite3 error") + else: + conn.commit() - stmt = "INSERT INTO sources (isbn, pzocode, full_name, short_name, abbr, descr, release_date, is_first_party, ogl_copyright_block) VALUES (?,?,?,?,?,?,?,?,?)" - try: - conn.executemany(stmt,inp_data) - except sqlite3.Error as e: - print("Error creating sources: {}".format(e)) - except: - print("Error creating sources something other than sqlite3 error") - else: - conn.commit() def do_traits(data, conn): - # create the two tables - table = """ + # create the two tables + table = """ CREATE TABLE traittype ( traittype_id INTEGER PRIMARY KEY, name TEXT NOT NULL ); """ - table_two = """ + table_two = """ CREATE TABLE trait ( trait_id INTEGER PRIMARY KEY, traittype_id INTEGER, @@ -92,40 +221,43 @@ CREATE TABLE trait ( FOREIGN KEY (traittype_id) REFERENCES traittype(traittype_id) ); """ - c = conn.cursor() - c.execute(table) - c.execute(table_two) - # insert data into traittype - inp_data = [] - for i in data['traittype']: - inp_data.append((i,)) # trailing comma necessary for one-item tuple - stmt = "INSERT INTO traittype (name) VALUES (?)" - try: - conn.executemany(stmt,inp_data) - except: - e = sys.exc_info()[0] - print("Error creating traittype: {}".format(e)) - print(vars(e)) - else: - conn.commit() - # insert data into trait - inp_data = [] - for i in data['trait']: - inp_data.append((i['descr'], i['name'], i['type'])) # trailing comma necessary for one-item tuple - stmt = "INSERT INTO trait (descr, short_name, traittype_id) SELECT ?,?, traittype_id FROM traittype WHERE traittype.name=?" - try: - conn.executemany(stmt,inp_data) - except sqlite3.Error as e: - print("Error creating trait input: {}".format(e)) - except: - e = sys.exc_info()[0] - print("Error creating trait: {}".format(e)) - print(vars(e)) - else: - conn.commit() + c = conn.cursor() + c.execute(table) + c.execute(table_two) + # insert data into traittype + inp_data = [] + for i in data['traittype']: + inp_data.append((i, )) # trailing comma necessary for one-item tuple + stmt = "INSERT INTO traittype (name) VALUES (?)" + try: + conn.executemany(stmt, inp_data) + except: + e = sys.exc_info()[0] + print("Error creating traittype: {}".format(e)) + print(vars(e)) + else: + conn.commit() + # insert data into trait + inp_data = [] + for i in data['trait']: + inp_data.append( + (i['descr'], i['name'], + i['type'])) # trailing comma necessary for one-item tuple + stmt = "INSERT INTO trait (descr, short_name, traittype_id) SELECT ?,?, traittype_id FROM traittype WHERE traittype.name=?" + try: + conn.executemany(stmt, inp_data) + except sqlite3.Error as e: + print("Error creating trait input: {}".format(e)) + except: + e = sys.exc_info()[0] + print("Error creating trait: {}".format(e)) + print(vars(e)) + else: + conn.commit() + def do_size(data, conn): - table = """ + table = """ CREATE TABLE size ( size_id INTEGER PRIMARY KEY, short_name TEXT NOT NULL UNIQUE, @@ -135,99 +267,104 @@ CREATE TABLE size ( ); """ - c = conn.cursor() - c.execute(table) + c = conn.cursor() + c.execute(table) - inp_data = [] - for i in data: - inp_data.append((i['name'], i['space_in_ft'], i['reach_tall_ft'], i['reach_long_ft'])) + inp_data = [] + for i in data: + inp_data.append((i['name'], i['space_in_ft'], i['reach_tall_ft'], + i['reach_long_ft'])) + + stmt = "INSERT INTO size (short_name, space_in_ft, reach_tall_ft, reach_long_ft) VALUES (?,?,?,?)" + try: + conn.executemany(stmt, inp_data) + except: + print("Error creating size") + else: + conn.commit() - stmt = "INSERT INTO size (short_name, space_in_ft, reach_tall_ft, reach_long_ft) VALUES (?,?,?,?)" - try: - conn.executemany(stmt,inp_data) - except: - print("Error creating size") - else: - conn.commit() def do_weaponcategory(data, conn): - table = """ + table = """ CREATE TABLE weaponcategory ( weaponcategory_id INTEGER PRIMARY KEY, "name" TEXT NOT NULL UNIQUE ); """ - c = conn.cursor() - c.execute(table) + c = conn.cursor() + c.execute(table) - inp_data = [] - for i in data: - inp_data.append((i,)) # trailing comma necessary for one-item tuple + inp_data = [] + for i in data: + inp_data.append((i, )) # trailing comma necessary for one-item tuple + + stmt = "INSERT INTO weaponcategory(name) VALUES (?)" + try: + conn.executemany(stmt, inp_data) + except: + e = sys.exc_info()[0] + print("Error creating weaponcategory: {}".format(e)) + print(vars(e)) + else: + conn.commit() - stmt = "INSERT INTO weaponcategory(name) VALUES (?)" - try: - conn.executemany(stmt,inp_data) - except: - e = sys.exc_info()[0] - print("Error creating weaponcategory: {}".format(e)) - print(vars(e)) - else: - conn.commit() def do_movement(data, conn): - table = """ + table = """ CREATE TABLE movement ( movement_id INTEGER PRIMARY KEY, "name" TEXT UNIQUE NOT NULL ); """ - c = conn.cursor() - c.execute(table) + c = conn.cursor() + c.execute(table) - inp_data = [] - for i in data: - inp_data.append((i,)) # trailing comma necessary for one-item tuple + inp_data = [] + for i in data: + inp_data.append((i, )) # trailing comma necessary for one-item tuple + + stmt = "INSERT INTO movement(name) VALUES (?)" + try: + conn.executemany(stmt, inp_data) + except: + e = sys.exc_info()[0] + print("Error creating movement: {}".format(e)) + print(vars(e)) + else: + conn.commit() - stmt = "INSERT INTO movement(name) VALUES (?)" - try: - conn.executemany(stmt,inp_data) - except: - e = sys.exc_info()[0] - print("Error creating movement: {}".format(e)) - print(vars(e)) - else: - conn.commit() def do_frequency(data, conn): - table = """ + table = """ CREATE TABLE frequency ( freq_id INTEGER PRIMARY KEY, freq_descr TEXT NOT NULL UNIQUE ); """ - c = conn.cursor() - c.execute(table) + c = conn.cursor() + c.execute(table) - inp_data = [] - for i in data: - inp_data.append((i,)) # trailing comma necessary for one-item tuple + inp_data = [] + for i in data: + inp_data.append((i, )) # trailing comma necessary for one-item tuple + + stmt = "INSERT INTO frequency(freq_descr) VALUES (?)" + try: + conn.executemany(stmt, inp_data) + except: + e = sys.exc_info()[0] + print("Error creating frequency: {}".format(e)) + print(vars(e)) + else: + conn.commit() - stmt = "INSERT INTO frequency(freq_descr) VALUES (?)" - try: - conn.executemany(stmt,inp_data) - except: - e = sys.exc_info()[0] - print("Error creating frequency: {}".format(e)) - print(vars(e)) - else: - conn.commit() def do_alignment(data, conn): - # print(data) - table = """ + # print(data) + table = """ CREATE TABLE alignment ( alignment_id INTEGER PRIMARY KEY, "name" TEXT UNIQUE NOT NULL, -- 'Lawful Good' @@ -235,48 +372,50 @@ CREATE TABLE alignment ( ); """ - c = conn.cursor() - c.execute(table) + c = conn.cursor() + c.execute(table) - inp_data = [] - for i in data: - inp_data.append((i['name'], i['abbr'])) + inp_data = [] + for i in data: + inp_data.append((i['name'], i['abbr'])) + + stmt = "INSERT INTO alignment(name, abbr) VALUES (?,?)" + try: + conn.executemany(stmt, inp_data) + except: + print("Error creating alignment") + else: + conn.commit() - stmt = "INSERT INTO alignment(name, abbr) VALUES (?,?)" - try: - conn.executemany(stmt,inp_data) - except: - print("Error creating alignment") - else: - conn.commit() def do_langrarity(data, conn): - table = """ + table = """ CREATE TABLE langrarity ( rarity_id INTEGER PRIMARY KEY, rarity_name TEXT NOT NULL UNIQUE ); """ - c = conn.cursor() - c.execute(table) + c = conn.cursor() + c.execute(table) - inp_data = [] - for i in data: - inp_data.append((i,)) # trailing comma necessary for one-item tuple + inp_data = [] + for i in data: + inp_data.append((i, )) # trailing comma necessary for one-item tuple + + stmt = "INSERT INTO langrarity(rarity_name) VALUES (?)" + try: + conn.executemany(stmt, inp_data) + except: + e = sys.exc_info()[0] + print("Error creating langrarity: {}".format(e)) + print(vars(e)) + else: + conn.commit() - stmt = "INSERT INTO langrarity(rarity_name) VALUES (?)" - try: - conn.executemany(stmt,inp_data) - except: - e = sys.exc_info()[0] - print("Error creating langrarity: {}".format(e)) - print(vars(e)) - else: - conn.commit() def do_actioncost(data, conn): - table = """ + table = """ CREATE TABLE actioncost ( actioncost_id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, @@ -284,23 +423,24 @@ CREATE TABLE actioncost ( ); """ - c = conn.cursor() - c.execute(table) + c = conn.cursor() + c.execute(table) - inp_data = [] - for i in data: - inp_data.append((i['name'], i['abbr'])) + inp_data = [] + for i in data: + inp_data.append((i['name'], i['abbr'])) + + stmt = "INSERT INTO actioncost(name, abbr) VALUES (?,?)" + try: + conn.executemany(stmt, inp_data) + except: + print("Error creating actioncost") + else: + conn.commit() - stmt = "INSERT INTO actioncost(name, abbr) VALUES (?,?)" - try: - conn.executemany(stmt,inp_data) - except: - print("Error creating actioncost") - else: - conn.commit() def do_abilityscore(data, conn): - table = """ + table = """ CREATE TABLE abilityscore ( abilityscore_id INTEGER PRIMARY KEY, flag_rep INTEGER NOT NULL, @@ -309,37 +449,38 @@ CREATE TABLE abilityscore ( ); """ - c = conn.cursor() - c.execute(table) + c = conn.cursor() + c.execute(table) - inp_data = [] - for i in data: - inp_data.append((i['flag_rep'], i['short_name'], i['long_name'])) + inp_data = [] + for i in data: + inp_data.append((i['flag_rep'], i['short_name'], i['long_name'])) + + stmt = "INSERT INTO abilityscore (flag_rep, short_name, long_name) VALUES (?,?,?)" + try: + conn.executemany(stmt, inp_data) + except: + print("Error creating abilityscore") + else: + conn.commit() - stmt = "INSERT INTO abilityscore (flag_rep, short_name, long_name) VALUES (?,?,?)" - try: - conn.executemany(stmt,inp_data) - except: - print("Error creating abilityscore") - else: - conn.commit() def get_db_conn(): - ## Get database connection - conn = sqlite3.connect(DBFILE) # eventually hook this up to be the main db - ## Set pragmas - pragma1 = "PRAGMA synchronous=OFF;" - pragma2 = "PRAGMA count_changes=OFF;" - pragma3 = "PRAGMA journal_mode=MEMORY;" - pragma4 = "PRAGMA temp_store=MEMORY;" - pragma5 = "PRAGMA foreign_keys=ON;" - conn.execute(pragma1) - conn.execute(pragma2) - conn.execute(pragma3) - conn.execute(pragma4) - conn.execute(pragma5) - return conn + ## Get database connection + conn = sqlite3.connect(DBFILE) # eventually hook this up to be the main db + ## Set pragmas + pragma1 = "PRAGMA synchronous=OFF;" + pragma2 = "PRAGMA count_changes=OFF;" + pragma3 = "PRAGMA journal_mode=MEMORY;" + pragma4 = "PRAGMA temp_store=MEMORY;" + pragma5 = "PRAGMA foreign_keys=ON;" + conn.execute(pragma1) + conn.execute(pragma2) + conn.execute(pragma3) + conn.execute(pragma4) + conn.execute(pragma5) + return conn if __name__ == "__main__": - main() + main() diff --git a/data/yaml/skills.yaml b/data/yaml/skills.yaml new file mode 100644 index 0000000..7d161bd --- /dev/null +++ b/data/yaml/skills.yaml @@ -0,0 +1,87 @@ +# TODO all the descriptions +skill: + - name: Acrobatics + source: + - abbr: CRB + page_start: 240 + descr: TODO + - name: Arcana + source: + - abbr: CRB + page_start: 241 + descr: TODO + - name: Athletics + source: + - abbr: CRB + page_start: 241 + descr: TODO + - name: Crafting + source: + - abbr: CRB + page_start: 243 + descr: TODO + - name: Deception + source: + - abbr: CRB + page_start: 245 + descr: TODO + - name: Diplomacy + source: + - abbr: CRB + page_start: 246 + descr: TODO + - name: Intimidation + source: + - abbr: CRB + page_start: 247 + descr: TODO + - name: Lore + source: + - abbr: CRB + page_start: 247 + descr: TODO + - name: Medicine + source: + - abbr: CRB + page_start: 248 + descr: TODO + - name: Nature + source: + - abbr: CRB + page_start: 249 + descr: TODO + - name: Occultism + source: + - abbr: CRB + page_start: 249 + descr: TODO + - name: Performance + source: + - abbr: CRB + page_start: 250 + descr: TODO + - name: Religion + source: + - abbr: CRB + page_start: 250 + descr: TODO + - name: Society + source: + - abbr: CRB + page_start: 250 + descr: TODO + - name: Stealth + source: + - abbr: CRB + page_start: 251 + descr: TODO + - name: Survival + source: + - abbr: CRB + page_start: 252 + descr: TODO + - name: Thievery + source: + - abbr: CRB + page_start: 253 + descr: TODO