import zipfile
import re
import urllib3
from io import BytesIO
from xml.etree import ElementTree as ET
import pandas as pd
import df2gspread.df2gspread as d2gIPC schema parsing
The International Patent Classification is a hierarchical classification system that assigns technological domains to patents. It is useful to calculate indicators on the technological activities of inventors.
The IPC schema is published by the WIPO in the form of a very complex XML file. In this notebook, we extract the information on each of the elements of the classification system.
Get IPC schema
Here, we read directly the remote zip file, without having to download it and store it locally.
# Update this string when WIPO releases a new schema version.
# Check https://www.wipo.int/ipc/en/ for the latest edition date (format: YYYYMMDD).
ipc_schema_version = "20250101"
# this URL changes when there are updates, so it should be updated periodically
url = f"https://www.wipo.int/ipc/itos4ipc/ITSupport_and_download_area/{ipc_schema_version}/MasterFiles/ipc_scheme_{ipc_schema_version}.zip"
http = urllib3.PoolManager()
response = http.request("GET", url)
z = zipfile.ZipFile(BytesIO(response.data))
xml_filename = [name for name in z.namelist() if name.endswith(".xml")][0]
with z.open(xml_filename) as f:
tree = ET.parse(f)
root = tree.getroot()Parsing the tree
The text of the description of the nodes is contained in a series of text objects, which are children of ipcEntry objects. At the deepest level of the hierarchy, the text contains only a specification of a previously described item. Therefore, to obtain a meaningful string that describes an individual node of the bottom of the tree, you have to get the text of each of the parents.
Tree structure
The first useful thing is to write a function that iterates over the specific kind of nodes that we want to examine, which are the ipcEntry nodes.
default_excluded_kind = ('i', 'g', 'n', 't')
def node_filter(node, excluded_kind_list = default_excluded_kind):
if 'kind' not in node.attrib:
return False
if node.attrib['kind'] not in excluded_kind_list:
return True
else:
return False
def node_iterator(tree, excluded_kind_list = default_excluded_kind):
for node in tree.iter('{http://www.wipo.int/classifications/ipc/masterfiles}ipcEntry'):
if node_filter(node, excluded_kind_list):
yield nodeText of a node
Nodes contain multiple text objects, and the full description is obtained by a concatenation with ';. This is a function that obtains the full text of a node.
def get_ipcEntry_text(ipcEntry, sep='; '):
# initialise the output list that contains the text elements
txt = []
# when we look at an element, it is always a list, the first element of which
# contains the useful text object
for sub in ipcEntry[0].findall('.//{http://www.wipo.int/classifications/ipc/masterfiles}text') :
# get the text of the node and check whether it is None
this_txt = sub.text
if this_txt:
txt.append(this_txt)
# finally, join the text pieces with the separator
return sep.join(txt)# next, we map all the children of each node and get the text and title of each one
children_map = {}
for node in node_iterator(scheme):
children = []
for child in node:
if not node_filter(child):
continue
children.append(child.attrib['symbol'])
children_map[node.attrib['symbol']] = {
'children':children,
'kind':node.attrib['kind'],
'text':get_ipcEntry_text(node)
}# this dataframe contains all the information we need
node_df = pd.DataFrame(children_map).transpose()# get a series with the parent of a given node
parent_map = node_df.children.explode().\
dropna().reset_index().rename(columns={'children':'child', 'index':'parent'}).\
set_index('child').parent# this is a recursive function that gets all the parent symbols
def get_parent_symbols(symbol, parents=[]):
try:
parent_symbol = parent_map[symbol]
parents.append(parent_symbol)
return get_parent_symbols(parent_symbol, parents)
except KeyError:
return parents
get_parent_symbols('A01B0003280000')['A01B0003240000', 'A01B0003000000', 'A01B', 'A01', 'A']exclude_kinds = ('s', 'c')
# the full_texts will contain the full description of the symbols. For convenience, we start at
# the subgroup level
full_texts = {}
for symbol, symbol_data in node_df.iterrows():
# get the text of the current node, initialising the output data structure
full_text = [symbol_data['text']]
if symbol_data['kind'] not in exclude_kinds:
# get the text of the parent symbols
my_symbols = []
parent_symbols = get_parent_symbols(symbol, my_symbols)
parent_texts = [node_df.loc[parent_symbol, 'text'] \
for parent_symbol in parent_symbols \
if node_df.loc[parent_symbol, 'kind'] not in exclude_kinds]
# append the text to the full text
full_text.extend(parent_texts)
# we now create the string by joining the descriptions of the parent symbols, in reverse order
full_texts[symbol] = ': '.join(full_text[::-1])# this data structure contains the full description of a symbol
full_texts_s = pd.Series(full_texts).rename('full_text')node_df['full_text'] = full_texts_sFinally, to be able to perform join operations between the symbol descriptions and the symbols as they appear in PATSTAT, we need to unpack the symbols. As per the schema documentation, the symbols are defined as:
SCCUMMMMGGGGGG
With: - S = section: one uppercase letter - C = class: two digits - U = sub-class: one uppercase letter - M = main-group: four digits - G = group: six digits
def parse_subsubsymbol(s):
parsed_s = s[:2]
for c in s[2:]:
if c == '0':
break
else:
parsed_s += c
return parsed_s
def parse_subsymbol(s):
parsed_s = ''
for i, c in enumerate(s):
if c == '0':
parsed_s += ' '
else:
return parsed_s + s[i:]
def unpack_ipc_symbol(symbol):
if len(symbol) == 14:
main_symbol = symbol[:4]
sub_symbol = symbol[4:8]
subsub_symbol = symbol[8:]
return main_symbol + parse_subsymbol(sub_symbol) + '/' + parse_subsubsymbol(subsub_symbol)
else:
return symbol
# get the final results
final = full_texts_s.reset_index()
final['symbol'] = final['index'].apply(unpack_ipc_symbol)
final = final[['symbol', 'full_text']]# write to file
final.to_csv('ipc_descriptions.csv', index=False)# Replace YOUR_SHEET_ID with the ID of your own Google Sheet
# (the long string in the sheet URL between /d/ and /edit).
# The sheet must already exist and your Google account must have edit access.
datasheet = "YOUR_SHEET_ID"
d2g.upload(final, datasheet, f"Schema", row_names=False, col_names=True, clean=False)Symbol map
It is useful to have a mapping of the symbol names in the schema and in the PATSTAT database.
symbol_list = node_df.reset_index().rename(columns={'index':'symbol'}).symbol.to_list()
symbol_map = pd.Series(index=symbol_list, data=[unpack_ipc_symbol(s) for s in symbol_list])
symbol_mapA A
A01 A01
A01B A01B
A01B0001000000 A01B 1/00
A01B0001020000 A01B 1/02
...
H10N0097000000 H10N 97/00
H10N0099000000 H10N 99/00
H99 H99
H99Z H99Z
H99Z0099000000 H99Z 99/00
Length: 79473, dtype: objectNode children dataframe
# first, we create a data structure that accommodates our data in raw form
node_children = node_df.children# this is a recursive function that allows to obtain all the children of a given symbol (in schema format)
def get_all_children(tree, node):
# using the `get` funtion on a pd.Series object is a smart way of
# not having to explicitly deal with the case in which the value
# is not found
children = tree.get(node, [])
# now process recursively all the children that we found
all_children = []
for child in children:
all_children.append(child)
all_children.extend(get_all_children(tree, child))
return all_children
get_all_children(node_children, 'A01B0003040000')['A01B0003060000',
'A01B0003080000',
'A01B0003100000',
'A01B0003120000',
'A01B0003140000',
'A01B0003160000',
'A01B0003180000',
'A01B0003200000',
'A01B0003220000']# get now all the children of all the nodes
all_children = pd.Series(
index=node_df.index,
data=[get_all_children(node_df.children, node) for node in node_df.index]).rename('all_children')# for the purpose of performing analysis of perimeters, what we ultimately need is a list of
# children nodes in PATSTAT format
all_children_patstat = pd.DataFrame({
'patstat_symbol' : symbol_map,
'children' : all_children.apply(lambda children: ';'.join([unpack_ipc_symbol(child) for child in children]))
}).reset_index().rename(columns={'index':'schema_symbol'})# upload. Note that we upload only the symbols that have a long length. In fact, to obtain the children
# of the higher level nodes, one can perform a simple search on the string (i.e. A01B*)
datasheet = "YOUR_SHEET_ID" # same sheet ID as above
d2g.upload(all_children_patstat.loc[all_children_patstat.schema_symbol.apply(len)==14, :],
datasheet, "Children", row_names=False, col_names=True)