{"id":1163,"date":"2024-04-30T18:44:31","date_gmt":"2024-04-30T10:44:31","guid":{"rendered":"http:\/\/www.max-shu.com\/blog\/?p=1163"},"modified":"2024-04-30T18:44:32","modified_gmt":"2024-04-30T10:44:32","slug":"spider-flow%e7%88%ac%e8%99%ab%e4%bd%bf%e7%94%a8","status":"publish","type":"post","link":"http:\/\/www.max-shu.com\/blog\/?p=1163","title":{"rendered":"spider flow\u722c\u866b\u4f7f\u7528"},"content":{"rendered":"\n

\u4ee3\u7801\uff1a<\/strong><\/p>\n\n\n\n

\u4e0b\u8f7d20240417\u7684spider flow\u7684master\u5206\u652f\u7684\u4ee3\u7801\uff1a https:\/\/github.com\/ssssssss-team\/spider-flow<\/a><\/p>\n\n\n\n

\u4e0b\u8f7d20240429\u7684selenium\u63d2\u4ef6\u7684master\u5206\u652f\u7684\u4ee3\u7801\uff1a https:\/\/gitee.com\/ssssssss-team\/spider-flow-selenium<\/a><\/p>\n\n\n\n

\u5047\u5b9a\u5728JDK 21\u4e0b\u7f16\u8bd1\u548c\u8fd0\u884c\uff0cgithub\u4e0a\u7684\u4ee3\u7801\u539f\u6765\u662f\u5728JDK 1.8\u4e0a\u7f16\u8bd1\u548c\u8fd0\u884c\u7684\uff0c\u6240\u4ee5\u9700\u8981\u4fee\u6539\u3002<\/p>\n\n\n\n

\u89e3\u538b<\/strong>\uff1a<\/p>\n\n\n\n

    \u628aspider-flow-master.zip\u89e3\u538b\u5230\u7535\u8111\u4e0a\u7684\u67d0\u4e2a\u76ee\u5f55\u6bd4\u5982D:\\idealU_PROJECT\\spider-flow-master_20240417\uff0c\u7136\u540e\u628aspider-flow-selenium-master.zip\u89e3\u538b\u5230D:\\idealU_PROJECT\\spider-flow-master_20240417\\spider-flow-selenium-master_20240429\u76ee\u5f55\u4e0b\u3002<\/p>\n\n\n\n

\u4fee\u6539<\/strong>\uff1aspider-flow-core\\src\\main\\java\\org\\spiderflow\\core\\script\\ScriptManager.java<\/p>\n\n\n\n

\/\/<\/em>import jdk.nashorn.api.scripting.ScriptObjectMirror;<\/em><\/p>\n\n\n\n

import org.openjdk.nashorn.api.scripting.ScriptObjectMirror;<\/p>\n\n\n\n

\u4fee\u6539<\/strong>\uff1aspider-flow-core\\src\\main\\java\\org\\spiderflow\\core\\service\\SpiderFlowService.java<\/p>\n\n\n\n

import org.springframework.context.annotation.Lazy;<\/p>\n\n\n\n

…<\/p>\n\n\n\n

@Lazy<\/p>\n\n\n\n

@Autowired<\/p>\n\n\n\n

private SpiderJobManager spiderJobManager;<\/p>\n\n\n\n

\u4fee\u6539<\/strong>\uff1aspider-flow-core\/src\/main\/java\/org\/spiderflow\/core\/executor\/shape\/ExecuteSQLExecutor.java ??<\/p>\n\n\n\n

\u628a Array.get<\/em>(object,Math.min<\/em>(-1,index));<\/p>\n\n\n\n

\u6539\u6210\uff1aArray.get<\/em>(object,Math.min<\/em>(size-1,index));<\/p>\n\n\n\n

\u4fee\u6539<\/strong>\uff1aspider-flow-web\/src\/main\/java\/org\/spiderflow\/configuration\/ResourcesConfiguration.java<\/p>\n\n\n\n

public void addCorsMappings(CorsRegistry registry) {<\/p>\n\n\n\n

    registry.addMapping(“\/**”)<\/p>\n\n\n\n

    \/\/.allowedOrigins(“*”)<\/em><\/p>\n\n\n\n

    .allowedOriginPatterns(“*”)<\/p>\n\n\n\n

\u4fee\u6539<\/strong>\uff1aspider-flow-selenium-master_20240429\\src\\main\\java\\org\\spiderflow\\selenium\\driver\\ChromeDriverProvider.java<\/p>\n\n\n\n

    public WebDriver getWebDriver(SpiderNode node, String proxyStr) {<\/p>\n\n\n\n

        System.setProperty(“webdriver.chrome.driver”, chromeDriverPath);<\/p>\n\n\n\n

        ChromeOptions options = new ChromeOptions();<\/p>\n\n\n\n

\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0String userAgent = node.getStringJsonValue(USER_AGENT);<\/p>\n\n\n\n

\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\/\/\u5141\u8bb8\u8fdc\u7a0b\u8fde\u63a5\u8fdb\u884c\u8c03\u8bd5<\/p>\n\n\n\n

\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0options.addArguments(“–remote-allow-origins=*”);<\/p>\n\n\n\n

\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\/\/options.addArguments(“–headless”);<\/p>\n\n\n\n

\u4fee\u6539<\/strong>\uff1aspider-flow-selenium-master_20240429\\src\\main\\java\\org\\spiderflow\\selenium\\driver\\HtmlUnitDriverProvider.java<\/p>\n\n\n\n

import org.openqa.selenium.Platform;<\/p>\n\n\n\n

…<\/p>\n\n\n\n

\u00a0\u00a0\u00a0\u00a0public WebDriver getWebDriver(SpiderNode node, String proxyStr) {<\/p>\n\n\n\n

\u00a0 \/\/DesiredCapabilities capabilities = DesiredCapabilities.htmlUnit();<\/p>\n\n\n\n

\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0DesiredCapabilities capabilities = new DesiredCapabilities(“htmlunit”, “”, Platform.ANY);<\/p>\n\n\n\n

<\/p>\n\n\n\n

\u4fee\u6539<\/strong>\uff1apom.xml<\/p>\n\n\n\n

<parent><\/p>\n\n\n\n

    <groupId>org.springframework.boot<\/groupId><\/p>\n\n\n\n

    <artifactId>spring-boot-starter-parent<\/artifactId><\/p>\n\n\n\n

    <version>2.7.18<\/version><\/p>\n\n\n\n

<\/parent><\/p>\n\n\n\n

…<\/p>\n\n\n\n

    <guava.version>31.1-jre<\/guava.version><\/p>\n\n\n\n

…<\/p>\n\n\n\n

<dependencies><\/p>\n\n\n\n

    <dependency><\/p>\n\n\n\n

        <groupId>org.openjdk.nashorn<\/groupId><\/p>\n\n\n\n

        <artifactId>nashorn-core<\/artifactId><\/p>\n\n\n\n

        <version>15.4<\/version><\/p>\n\n\n\n

    <\/dependency><\/p>\n\n\n\n

    <dependency><\/p>\n\n\n\n

        <groupId>mysql<\/groupId><\/p>\n\n\n\n

        <artifactId>mysql-connector-java<\/artifactId><\/p>\n\n\n\n

        <version>8.0.33<\/version><\/p>\n\n\n\n

    <\/dependency><\/p>\n\n\n\n

…<\/p>\n\n\n\n

    <module>spider-flow-web<\/module><\/p>\n\n\n\n

    <module>spider-flow-selenium-master_20240429<\/module><\/p>\n\n\n\n

<\/modules><\/p>\n\n\n\n

\u4fee\u6539<\/strong>\uff1aspider-flow-web\/pom.xml\uff1a<\/p>\n\n\n\n

        <dependency><\/p>\n\n\n\n

            <groupId>org.seleniumhq.selenium<\/groupId><\/p>\n\n\n\n

            <artifactId>selenium-java<\/artifactId><\/p>\n\n\n\n

            <version>4.20.0<\/version><\/p>\n\n\n\n

        <\/dependency><\/p>\n\n\n\n

        <!– \u5f15\u5165selenium\u63d2\u4ef6 –><\/p>\n\n\n\n

        <dependency><\/p>\n\n\n\n

            <groupId>org.spiderflow<\/groupId><\/p>\n\n\n\n

            <artifactId>spider-flow-selenium<\/artifactId><\/p>\n\n\n\n

        <\/dependency><\/p>\n\n\n\n

<\/dependencies><\/p>\n\n\n\n

…<\/p>\n\n\n\n

    <finalName>spider-flow-${project.version}<\/finalName><\/p>\n\n\n\n

    <mainClass>org.spiderflow.SpiderApplication<\/mainClass><\/p>\n\n\n\n

\u4fee\u6539<\/strong>\uff1aspider-flow-selenium-master_20240429\/pom.xml<\/p>\n\n\n\n

        <dependency><\/p>\n\n\n\n

            <groupId>org.seleniumhq.selenium<\/groupId><\/p>\n\n\n\n

            <artifactId>selenium-java<\/artifactId><\/p>\n\n\n\n

            <version>4.20.0<\/version><\/p>\n\n\n\n

        <\/dependency><\/p>\n\n\n\n

        <dependency><\/p>\n\n\n\n

            <groupId>org.seleniumhq.selenium<\/groupId><\/p>\n\n\n\n

            <artifactId>htmlunit-driver<\/artifactId><\/p>\n\n\n\n

            <version>4.13.0<\/version><\/p>\n\n\n\n

        <\/dependency><\/p>\n\n\n\n

<\/dependencies><\/p>\n\n\n\n

\u4fee\u6539<\/strong>\uff1aspider-flow-web\/src\/main\/resources\/application.properties<\/p>\n\n\n\n

server.port=8088<\/p>\n\n\n\n

spring.profiles.active=dev<\/p>\n\n\n\n

logging.level.root=INFO<\/p>\n\n\n\n

#logging.level.org.spiderflow=DEBUG<\/em><\/p>\n\n\n\n

#<\/em>\u5e73\u53f0\u6700\u5927\u7ebf\u7a0b\u6570<\/em><\/p>\n\n\n\n

spider.thread.max=64<\/p>\n\n\n\n

#<\/em>\u5355\u4efb\u52a1\u9ed8\u8ba4\u6700\u5927\u7ebf\u7a0b\u6570<\/em><\/p>\n\n\n\n

spider.thread.default=4<\/p>\n\n\n\n

#<\/em>\u8bbe\u7f6e\u4e3a<\/em>true<\/em>\u65f6\u5b9a\u65f6\u4efb\u52a1\u624d\u751f\u6548<\/em><\/p>\n\n\n\n

spider.job.enable=true<\/p>\n\n\n\n

#<\/em>\u722c\u866b\u4efb\u52a1\u7684\u5de5\u4f5c\u7a7a\u95f4<\/em><\/p>\n\n\n\n

spider.workspace=spider_workspace<\/p>\n\n\n\n

#<\/em>\u5e03\u9686\u8fc7\u6ee4\u5668\u9ed8\u8ba4\u5bb9\u91cf<\/em><\/p>\n\n\n\n

spider.bloomfilter.capacity=1000000<\/p>\n\n\n\n

#<\/em>\u5e03\u9686\u8fc7\u6ee4\u5668\u9ed8\u8ba4\u5bb9\u9519\u7387<\/em><\/p>\n\n\n\n

spider.bloomfilter.error-rate=0.0001<\/p>\n\n\n\n

#<\/em>\u6b7b\u5faa\u73af\u68c0\u6d4b<\/em>(<\/em>\u8282\u70b9\u6267\u884c\u6b21\u6570\u8d85\u8fc7\u8be5\u503c\u65f6\u8ba4\u4e3a\u662f\u6b7b\u5faa\u73af<\/em>)<\/em>\u9ed8\u8ba4\u503c\u4e3a<\/em>5000<\/em><\/p>\n\n\n\n

#spider.detect.dead-cycle=5000<\/em><\/p>\n\n\n\n

spring.jackson.date-format=yyyy-MM-dd HH:mm:ss<\/p>\n\n\n\n

spring.jackson.time-zone=GMT+8<\/p>\n\n\n\n

spring.jackson.serialization.fail_on_empty_beans<\/em>=false<\/p>\n\n\n\n

spring.mvc.favicon.enabled=false<\/p>\n\n\n\n

spring.datasource.driver-class-name=com.mysql.jdbc.Driver<\/p>\n\n\n\n

spring.datasource.username=${my.db.user}<\/p>\n\n\n\n

spring.datasource.password=${my.db.passwd}<\/p>\n\n\n\n

spring.datasource.url=jdbc:mysql:\/\/${my.db.host}:${my.db.port}\/spiderflow?useSSL=false&useUnicode=true&characterEncoding=UTF8&autoReconnect=true<\/p>\n\n\n\n

#JavaMailSender<\/em>\u90ae\u4ef6\u53d1\u9001\u7684\u914d\u7f6e<\/em><\/p>\n\n\n\n

spring.mail.protocol=smtp<\/p>\n\n\n\n

spring.mail.host=smtp.exmail.qq.com<\/a><\/p>\n\n\n\n

spring.mail.port=465<\/p>\n\n\n\n

spring.mail.username=xxx@qq.com<\/a><\/p>\n\n\n\n

spring.mail.password=xxxyyy<\/p>\n\n\n\n

spring.mail.default-encoding=UTF-8<\/p>\n\n\n\n

spring.mail.properties.mail.smtp.auth=true<\/p>\n\n\n\n

spring.mail.properties.mail.smtp.starttls.enable=true<\/p>\n\n\n\n

spring.mail.properties.mail.smtp.starttls.required=true<\/p>\n\n\n\n

spring.mail.properties.mail.smtp.socketFactory.class=javax.net.ssl.SSLSocketFactory<\/a><\/p>\n\n\n\n

spring.mail.properties.mail.smtp.socketFactory.port=465<\/p>\n\n\n\n

spring.mail.properties.mail.smtp.socketFactory.fallback=false<\/p>\n\n\n\n

spring.mail.properties.mail.smtp.ssl.enable=true<\/p>\n\n\n\n

spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration,org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration<\/p>\n\n\n\n

#selenium<\/em>\u63d2\u4ef6\u914d\u7f6e<\/em><\/p>\n\n\n\n

#<\/em>\u8bbe\u7f6e<\/em>chrome<\/em>\u7684<\/em>WebDriver<\/em>\u9a71\u52a8\u8def\u5f84\uff0c\u4e0b\u8f7d\u5730\u5740\uff1a<\/em>http:\/\/npm.taobao.org\/mirrors\/chromedriver\/<\/a>\uff0c\u6ce8\u610f\u7248\u672c\u95ee\u9898<\/em><\/p>\n\n\n\n

selenium.driver.chrome=${spider.workspace}\\\\chromedriver.exe<\/p>\n\n\n\n

#<\/em>\u8bbe\u7f6e<\/em>fireFox<\/em>\u7684<\/em>WebDriver<\/em>\u9a71\u52a8\u8def\u5f84\uff0c\u4e0b\u8f7d\u5730\u5740\uff1a<\/em>https:\/\/github.com\/mozilla\/geckodriver\/releases<\/a><\/p>\n\n\n\n

selenium.driver.firefox=${spider.workspace}\\\\geckodriver.exe<\/p>\n\n\n\n

#<\/em>\u722c\u866b\u901a\u77e5\u76f8\u5173\u5185\u5bb9\u914d\u7f6e<\/em>,<\/em>\u53ef\u4f7f\u7528<\/em>SpiderFlow<\/em>\u4e2d\u7684\u53d8\u91cf\u540d\u548c\u4ee5\u4e0b\u53d8\u91cf\u540d<\/em>:currentDate:<\/em>\u5f53\u524d\u53d1\u9001\u65f6\u95f4<\/em><\/p>\n\n\n\n

spider.notice.subject=spider-flow\u6d41\u7a0b\u901a\u77e5<\/p>\n\n\n\n

spider.notice.content.start=\u6d41\u7a0b\u5f00\u59cb\u6267\u884c\uff1a{name}\uff0c\u5f00\u59cb\u65f6\u95f4\uff1a{currentDate}<\/p>\n\n\n\n

spider.notice.content.end=\u6d41\u7a0b\u6267\u884c\u5b8c\u6bd5\uff1a{name}\uff0c\u7ed3\u675f\u65f6\u95f4\uff1a{currentDate}<\/p>\n\n\n\n

spider.notice.content.exception=\u6d41\u7a0b\u53d1\u751f\u5f02\u5e38\uff1a{name}\uff0c\u5f02\u5e38\u65f6\u95f4\uff1a{currentDate}<\/p>\n\n\n\n

\u589e\u52a0<\/strong>\uff1aspider-flow-web\/src\/main\/resources\/application-dev.properties<\/p>\n\n\n\n

spider.thread.default=2<\/p>\n\n\n\n

spider.workspace=spider_workspace<\/p>\n\n\n\n

my.db.user=xxx<\/p>\n\n\n\n

my.db.passwd=xxxyyy<\/p>\n\n\n\n

my.db.host=192.168.1.248<\/p>\n\n\n\n

my.db.port=3306<\/p>\n\n\n\n

selenium.driver.chrome=${spider.workspace}\\\\chromedriver.exe<\/p>\n\n\n\n

selenium.driver.firefox=${spider.workspace}\\\\geckodriver.exe<\/p>\n\n\n\n

\u589e\u52a0<\/strong>\uff1aspider-flow-web\/src\/main\/resources\/application-test.properties<\/p>\n\n\n\n

spider.thread.default=4<\/p>\n\n\n\n

spider.workspace=\/data\/spider<\/p>\n\n\n\n

my.db.user=xxx<\/p>\n\n\n\n

my.db.passwd=xxxyyy<\/p>\n\n\n\n

my.db.host=192.168.1.249<\/p>\n\n\n\n

my.db.port=3306<\/p>\n\n\n\n

selenium.driver.chrome=${spider.workspace}\/chromedriver<\/p>\n\n\n\n

selenium.driver.firefox=${spider.workspace}\/geckodriver<\/p>\n\n\n\n

\u589e\u52a0<\/strong>\uff1aspider-flow-web\/src\/main\/resources\/application-prod.properties  \uff08\u7565\uff09<\/p>\n\n\n\n

\u914d\u7f6eSpiderApplication\u8fd0\u884c\u9009\u9879<\/strong>\uff0c\u589e\u52a0”Add VM options”\uff0c\u6dfb\u52a0\u8fd0\u884c\u65f6\u53c2\u6570\uff1a –add-opens java.base\/java.lang=ALL-UNNAMED<\/p>\n\n\n\n

spider-flow\u5b89\u88c5\uff1a<\/strong><\/p>\n\n\n\n

$ mkdir \/data\/spider\/<\/p>\n\n\n\n

$ cd \/data\/spider\/<\/p>\n\n\n\n

$ wget https:\/\/chromedriver.storage.googleapis.com\/124.0.6367.91\/chromedriver_linux64.zip<\/a><\/p>\n\n\n\n

$ unzip chromedriver_linux64.zip<\/p>\n\n\n\n

$ wget https:\/\/github.com\/mozilla\/geckodriver\/releases\/download\/v0.34.0\/geckodriver-v0.34.0-linux64.tar.gz<\/a><\/p>\n\n\n\n

$ tar -zxvf geckodriver-v0.34.0-linux64.tar.gz<\/p>\n\n\n\n

$ cd ~<\/p>\n\n\n\n

$ mkdir spiderflow\/<\/p>\n\n\n\n

$ cd spiderflow<\/p>\n\n\n\n

$ \u628aspider-flow.jar\u62f7\u8d1d\u5230\u672c\u76ee\u5f55\u3002<\/p>\n\n\n\n

$ \u5230mysql\u91cc\u9762\u6267\u884cspiderflow\u7684db\u76ee\u5f55\u4e0b\u7684spiderflow.sql\uff0c\u5efa\u7acbspiderflow\u6570\u636e\u5e93\u3002<\/p>\n\n\n\n

\u8fd0\u884c\uff1a<\/strong><\/p>\n\n\n\n

$ nohup java -Xmn64m -Xms256m -Xmx256m –add-opens java.base\/java.lang=ALL-UNNAMED -jar spider-flow.jar\u00a0 –spring.profiles.active=test \/dev\/null 2>&1 &<\/p>\n\n\n\n

\u751f\u6210\u7684log\u4f4d\u7f6e\uff08\u914d\u7f6e\u6587\u4ef6\u7684”spider.workspace”\u6240\u8bbe\u7f6e\u7684\u76ee\u5f55\u4e0b\uff09\uff1a<\/p>\n\n\n\n

$ ll \/data\/spider\/logs\/<\/p>\n\n\n\n

<\/p>\n\n\n\n

\u8bbf\u95eeWEB\uff1a<\/strong><\/p>\n\n\n\n

\u672c\u673a\u8bbf\u95ee\uff1a http:\/\/127.0.0.1:8088<\/p>\n\n\n\n

\u4f7f\u7528\uff1a<\/strong><\/p>\n\n\n\n

\u6559\u7a0b\uff1a https:\/\/www.bookstack.cn\/read\/spiderflow-0.5.0\/%E4%BD%BF%E7%94%A8%E6%95%99%E7%A8%8B.md<\/a><\/p>\n\n\n\n