Django-Scrapy实现生成后端json接口

django startproject 项目名称

cd 项目名称
python manage.py startapp appname

# cd django的根目录下
cd job_hnting
scrapy startproject 项目名称
#创建爬虫
scrapy genspider spidername 'www.xxx.com'

import os
import django

#导入
os.environ['DJANGO_SETTINGS_MODULE'] = 'job_hnting.settings'
#手动初始化
django.setup()

pip install scrapy_djangoitem

import scrapy
# 引入django中app中models文件中的类
from app51.models import app51data
# scrapy与django对接的库
from scrapy_djangoitem import DjangoItem


class JobprojectItem(DjangoItem):
  #引用django下的model中的类名
  django_model = app51data

# 定位数据位置，提取json数据
    search_pattern = "window.__SEARCH_RESULT__ = (.*&#63;)</script>"
    jsonText = re.search(search_pattern, response.text, re.M | re.S).group(1)

# 解析json数据
    jsonObject = json.loads(jsonText)
    number = jsonObject['total_page']

  for number in range(1,int(numbers)+1):
      next_page_url = self.url.format(self.name,number)
      # print(next_page_url)
      #构造的Urlcallback到data_parse函数中
      yield scrapy.Request(url=next_page_url,callback=self.data_parse)

 for job_item in jsonObject["engine_search_result"]:
      items = JobprojectItem()
      items['job_name'] = job_item['job_name']
      items['company_name'] = job_item["company_name"]
      # 发布时间
      items['Releasetime'] = job_item['issuedate']
      items['salary'] = job_item['providesalary_text']
      items['site'] = job_item['workarea_text']
      .......

class SeemeispiderPipeline(object):
  def process_item(self, item, spider):
    item.save()
    return item

DATABASES = {
  # 方法一
  'default': {
    'ENGINE': 'django.db.backends.mysql',  # 数据库引擎
    'NAME': 'mysite',            # 数据库名称
    'USER': 'root',           # 数据库登录用户名
    'PASSWORD': '123',        # 密码
    'HOST': '127.0.0.1',        # 数据库主机IP，如保持默认，则为127.0.0.1
    'PORT': 3306,              # 数据库端口号，如保持默认，则为3306
  }
}

[client]
database = blog
user = blog
password = blog
host =127.0.0.1
port = 3306
default-character-set = utf8

DATABASES = {
  # 方法二：
  'default': {
    'ENGINE': 'django.db.backends.mysql',
    'OPTIONS': {
      'read_default_file': 'utils/dbs/my.cnf',
    },
  }
}

import pymysql
pymysql.install_as_MySQLdb()

from django.db import models
# Create your models here.
#定义app51的数据模型
class app51data(models.Model):
  #发布时间,长度20
  Releasetime = models.CharField(max_length=20)
  #职位名，长度50
  job_name =models.CharField(max_length=50)
  #薪水
  salary = models.CharField(max_length=20)
  #工作地点
  site = models.CharField(max_length=50)
  #学历水平
  education = models.CharField(max_length=20)
  #公司名称
  company_name = models.CharField(max_length=50)
  #工作经验
  Workexperience = models.CharField(max_length=20)
  #指定表名
  class Meta:
    db_table = 'jobsql51'

  def __str__(self):
    return self.job_name

python manage.py makemigrations
python manage.py migrate

def last_executed_query(self, cursor, sql, params):
    # With MySQLdb, cursor objects have an (undocumented) "_executed"
    # attribute where the exact query sent to the database is saved.
    # See MySQLdb/cursors.py in the source distribution.
    query = getattr(cursor, '_executed', None)
    if query is not None:
      #query = query.decode(errors='replace')
      uery = query.encode(errors='replace')
    return query

from django.shortcuts import render
from django.http import HttpResponse
# Create your views here.
#引入数据
from .models import app51data
import json

def index(request):
  # return HttpResponse("hello world")
  # return render(request,'index.html')
  #获取所有的对象，转换成json格式
  data =app51data.objects.all()
  list3 = []
  i = 1
  for var in data:
    data = {}
    data['id'] = i
    data['Releasetime'] = var.Releasetime
    data['job_name'] = var.job_name
    data['salary'] = var.salary
    data['site'] = var.site
    data['education'] = var.education
    data['company_name'] = var.company_name
    data['Workexperience'] = var.Workexperience
    list3.append(data)
    i += 1

  # a = json.dumps(data)
  # b = json.dumps(list2)

  # 将集合或字典转换成json 对象
  c = json.dumps(list3)
  return HttpResponse(c)