電商商品概念與銷售知識圖譜

GoodsKG

　　GoodsKG, 基於京東網站的1300種商品上下級概念，約10萬商品品牌，約65萬品牌銷售關系，商品描述維度等知識庫，基於該知識庫可以支持商品屬性庫構建，商品銷售問答，品牌物品生產等知識查詢服務，也可用於情感分析等下游應用．

項目介紹

　　概念層級知識是整個常識知識體系中的重要組成部分。概念層級目前包括百科性概念層級和專業性概念層級兩類，百度百科概念體系以及互動百科分類體系是其中的一個代表。就后者而言，目前出現了許多垂直行業概念層級，如醫療領域概念體系，墊上領域概念體系，其中以淘寶、京東等為代表的電商網站在以商品為中心上，構建起了一種商品概念目錄層級以及商品與品牌，品牌與品牌之間的關聯關系。
　　本項目認為，電商網站中的商品分類目錄能夠供我們構建起一個商品概念體系，基於商品首頁，我們可以得到商品與商品品牌之間的關系，商品的屬性以及屬性的取值信息。基於這類信息，又可以進一步得到商品的畫像以及商品品牌的畫像。基於該畫像。可以對自然語言處理處理的幾個下游應用帶來幫助，如商品品牌識別，商品對象及屬性級別情感分析，商品評價短語庫構建，商品品牌競爭關系梳理等。
　　因此，本項目以京東電商為實驗數據來源，采集京東商品目錄樹，並獲取其對應的底層商品概念信息，組織形成商品知識圖譜。目前，該圖譜包括有概念的上下位is a關系以及商品品牌與商品之間的銷售sale關系共兩類關系，涉及商品概念數目1300+，商品品牌數目約10萬+，屬性數目幾千種，關系數目65萬規模。該項目可以進一步增強商品領域概念體系的應用，對自然語言處理處理的幾個下游應用帶來幫助，如商品品牌識別，商品對象及屬性級別情感分析，商品評價短語庫構建，商品品牌競爭關系梳理等提供基礎性的概念服務。

數據介紹

1, 基本數據內容 

 

 

2, is-a概念上下位關系 

MATCH p=()-[r:is_a]->() RETURN p LIMIT 25

3, sale銷售關系

MATCH p=()-[r:sales]->() RETURN p LIMIT 25

4, 混合關聯關系 

MATCH (n:Product) where n.name=’手機’ RETURN n LIMIT 25

 

GitHub源碼

https://github.com/chenjj9527/ProductKnowledgeGraph-master

#!/usr/bin/env python3
# coding: utf-8
# File: build_kg.py
# Author: cjj
# Date: 19-12-23

import urllib.request
from urllib.parse import quote_plus
from lxml import etree
import gzip
import chardet
import json
import pymongo

class GoodSchema:
    def __init__(self):
        self.conn = pymongo.MongoClient()
        return

    '''獲取搜索頁'''
    def get_html(self, url):
        headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/600.5.17 (KHTML, like Gecko) Version/8.0.5 Safari/600.5.17"}
        try:
            req = urllib.request.Request(url, headers=headers)
            data = urllib.request.urlopen(req).read()
            coding = chardet.detect(data)
            html = data.decode(coding['encoding'])
        except:
            req = urllib.request.Request(url, headers=headers)
            data = urllib.request.urlopen(req).read()
            html = data.decode('gbk')


        return html

    '''獲取詳情頁'''
    def get_detail_html(self, url):
        headers = {
            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
            "accept-encoding": "gzip, deflate, br",
            "accept-language": "en-US,en;q=0.9",
            "cache-control": "max-age=0",
            "referer": "https://www.jd.com/allSort.aspx",
            "upgrade-insecure-requests": 1,
            "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/66.0.3359.181 Chrome/66.0.3359.181 Safari/537.36"
        }
        try:
            req = urllib.request.Request(url, headers=headers)
            data = urllib.request.urlopen(req).read()
            html = gzip.decompress(data)
            coding = chardet.detect(html)
            html = html.decode(coding['encoding'])
        except Exception as e:
            req = urllib.request.Request(url, headers=headers)
            data = urllib.request.urlopen(req).read()
            html = gzip.decompress(data)
            html = html.decode('gbk')
        return html


    '''根據主頁獲取數據'''
    def home_list(self):
        url = 'https://www.jd.com/allSort.aspx'
        html = self.get_html(url)
        selector = etree.HTML(html)
        divs = selector.xpath('//div[@class= "category-item m"]')
        for indx, div in enumerate(divs):
            first_name = div.xpath('./div[@class="mt"]/h2/span/text()')[0]
            second_classes = div.xpath('./div[@class="mc"]/div[@class="items"]/dl')
            for dl in second_classes:
                second_name = dl.xpath('./dt/a/text()')[0]
                third_classes = ['https:' + i for i in dl.xpath('./dd/a/@href')]
                third_names = dl.xpath('./dd/a/text()')
                for third_name, url in zip(third_names, third_classes):
                    try:
                        attr_dict = self.parser_goods(url)
                        attr_brand = self.collect_brands(url)
                        attr_dict.update(attr_brand)
                        data = {}
                        data['fisrt_class'] = first_name
                        data['second_class'] = second_name
                        data['third_class'] = third_name
                        data['attrs'] = attr_dict
                        self.conn['goodskg']['data'].insert(data)
                        print(indx, len(divs), first_name, second_name, third_name)
                    except Exception as e:
                        print(e)
        return

    '''解析商品數據'''
    def parser_goods(self, url):
        html = self.get_detail_html(url)
        selector = etree.HTML(html)
        title = selector.xpath('//title/text()')
        attr_dict = {}
        other_attrs = ''.join([i for i in html.split('\n') if 'other_exts' in i])
        other_attr = other_attrs.split('other_exts =[')[-1].split('];')[0]
        if other_attr and 'var other_exts ={};' not in other_attr:
            for attr in other_attr.split('},'):
                if '}' not in attr:
                    attr = attr + '}'
                data = json.loads(attr)
                key = data['name']
                value = data['value_name']
                attr_dict[key] = value
        attr_divs = selector.xpath('//div[@class="sl-wrap"]')
        for div in attr_divs:
            attr_name = div.xpath('./div[@class="sl-key"]/span/text()')[0].replace('：','')
            attr_value = ';'.join([i.replace('  ','') for i in div.xpath('./div[@class="sl-value"]/div/ul/li/a/text()')])
            attr_dict[attr_name] = attr_value

        return attr_dict

    '''解析品牌數據'''
    def collect_brands(self, url):
        attr_dict = {}
        brand_url = url + '&sort=sort_rank_asc&trans=1&md=1&my=list_brand'
        html = self.get_html(brand_url)
        if 'html' in html:
            return attr_dict
        data = json.loads(html)
        brands = []

        if 'brands' in data and data['brands'] is not None:
            brands = [i['name'] for i in data['brands']]
        attr_dict['品牌'] = ';'.join(brands)

        return attr_dict



if __name__ == '__main__':
    handler = GoodSchema()
    handler.home_list()

#!/usr/bin/env python3
# coding: utf-8
# File: build_kg.py
# Author: cjj
# Date: 19-12-23

import json
import os
from py2neo import Graph, Node, Relationship


class GoodsKg:
    def __init__(self):
        cur = '/'.join(os.path.abspath(__file__).split('/')[:-1])
        self.data_path = os.path.join(cur, 'data/goods_info.json')
        self.g = Graph(
            host="127.0.0.1",  # neo4j 搭載服務器的ip地址，ifconfig可獲取到
            http_port=7474,  # neo4j 服務器監聽的端口號
            user="neo4j",  # 數據庫user name，如果沒有更改過，應該是neo4j
            password="111111")
        return

    '''讀取數據'''
    def read_data(self):
        rels_goods = []
        rels_brand = []
        goods_attrdict = {}
        concept_goods = set()
        concept_brand = set()
        count = 0
        for line in open(self.data_path,encoding='UTF-8'):
            count += 1
            print(count)
            line = line.strip()
            data = json.loads(line)
            first_class = data['fisrt_class'].replace("'",'')
            second_class = data['second_class'].replace("'",'')
            third_class = data['third_class'].replace("'",'')
            attr = data['attrs']
            concept_goods.add(first_class)
            concept_goods.add(second_class)
            concept_goods.add(third_class)
            rels_goods.append('@'.join([second_class, 'is_a', '屬於', first_class]))
            rels_goods.append('@'.join([third_class, 'is_a', '屬於', second_class]))

            if attr and '品牌' in attr:
                brands = attr['品牌'].split(';')
                for brand in brands:
                    brand = brand.replace("'",'')
                    concept_brand.add(brand)
                    rels_brand.append('@'.join([brand, 'sales', '銷售', third_class]))

            goods_attrdict[third_class] = {name:value for name,value in attr.items() if name != '品牌'}

        return concept_brand, concept_goods, rels_goods, rels_brand

    '''構建圖譜'''
    def create_graph(self):
        concept_brand, concept_goods, rels_goods, rels_brand = self.read_data()
        print('creating nodes....')
        self.create_node('Product', concept_goods)
        self.create_node('Brand', concept_brand)
        print('creating edges....')
        self.create_edges(rels_goods, 'Product', 'Product')
        self.create_edges(rels_brand, 'Brand', 'Product')
        return

    '''批量建立節點'''
    def create_node(self, label, nodes):
        pairs = []
        bulk_size = 1000
        batch = 0
        bulk = 0
        batch_all = len(nodes)//bulk_size
        print(batch_all)
        for node_name in nodes:
            sql = """CREATE(:%s {name:'%s'})""" % (label, node_name)
            pairs.append(sql)
            bulk += 1
            if bulk % bulk_size == 0 or bulk == batch_all+1:
                sqls = '\n'.join(pairs)
                self.g.run(sqls)
                batch += 1
                print(batch*bulk_size,'/', len(nodes), 'finished')
                pairs = []
        return


    '''構造圖譜關系邊'''
    def create_edges(self, rels, start_type, end_type):
        batch = 0
        count = 0
        for rel in set(rels):
            count += 1
            rel = rel.split('@')
            start_name = rel[0]
            end_name = rel[3]
            rel_type = rel[1]
            rel_name = rel[2]
            sql = 'match (m:%s), (n:%s) where m.name = "%s" and n.name = "%s" create (m)-[:%s{name:"%s"}]->(n)' %(start_type, end_type, start_name, end_name,rel_type,rel_name)
            try:
                self.g.run(sql)
            except Exception as e:
                print(e)
            if count%10 == 0:
                print(count)

        return


if __name__ =='__main__':
    handler = GoodsKg()
    handler.create_graph()