Application of jingdong Elasticsearch search | 7 clock

Sometimes all we need is just a new perspective. Sometimes we just need a new Angle

Search on Jd.com for Elasticsearch

The development environment

• Spring boot 2.4.2
• Elasticsearch 7.10.1
• lombok
• Parse the web page jsoup 1.10.2
• Alibaba fastjson 1.2.73
• JDK 1.8
• Integrated IDE idea
• elasticsearch-head

All development environment full stack self-learning community public number reply computer environment keywords can be obtained.

Project overview

pom.xml

      
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
        <groupId>cn.com.codingce</groupId>
        <artifactId>codingce-es</artifactId>
        <version>0.0.1 - the SNAPSHOT</version>
        <relativePath/> <! -- lookup parent from repository -->
    </parent>
    <groupId>cn.com.codingce</groupId>
    <artifactId>codingce-es-jd</artifactId>
    <version>0.0.1 - the SNAPSHOT</version>
    <name>codingce-es-jd</name>
    <description>Demo project for Spring Boot</description>

    <properties>
        <java.version>1.8</java.version>
        <! -- Specify consistency with local version -->
        <elasticsearch.version>7.10.1</elasticsearch.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-thymeleaf</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-devtools</artifactId>
            <scope>runtime</scope>
            <optional>true</optional>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-configuration-processor</artifactId>
            <optional>true</optional>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <optional>true</optional>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>

        <! -- Parsing web pages -->
        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.10.2</version>
        </dependency>
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.73</version>
            <scope>compile</scope>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
                <configuration>
                    <excludes>
                        <exclude>
                            <groupId>org.projectlombok</groupId>
                            <artifactId>lombok</artifactId>
                        </exclude>
                    </excludes>
                </configuration>
            </plugin>
        </plugins>
    </build>

</project>
Copy the code

The implementation code

ElasticSearchConfig

This is just a single Elasticsearch

/** * ElasticSearch configuration class * find the object, put it in Spring and use it **@author mxz
 */
@Configuration
public class ElasticSearchConfig {

    @Bean
    public RestHighLevelClient restHighLevelClient(a) {
        RestHighLevelClient client = new RestHighLevelClient(
                RestClient.builder(
                        new HttpHost("localhost".9200."http")));
// new HttpHost("localhost", 9201, "http")));
        returnclient; }}Copy the code

Entity class Content

It is used to correspond to the data of a single commodity of JINGdong

/** * Entity class **@author mxz
 */
@Data
@NoArgsConstructor
@AllArgsConstructor
public class Content {
    private String title;
    private String img;
    private String price;
}
Copy the code

Utility class HtmlParseUtil

Used to parse jd search data

/ * * *@author mxz
 */
@Component
public class HtmlParseUtil {

    public List<Content> parseJD(String keywords) throws Exception {

        / / get request at https://search.jd.com/Search?keyword=java
        String url = "https://search.jd.com/Search?keyword=" + keywords;

        // Parse the web page (return Document is the browser Document object)
        Document document = Jsoup.parse(new URL(url), 30000);

        // All the methods available in JS can be used here
        Element element = document.getElementById("J_goodsList");
// System.out.println(element.html());


        ArrayList<Content> goodList = new ArrayList<>();

        // Get all the li elements
        Elements elements = element.getElementsByTag("li");
        // Get the contents of the element
        for (Element el : elements) {
            // For sites with lots of these images, all the images are loaded lazily
            String img = el.getElementsByTag("img").eq(0).attr("data-lazy-img");
            String price = el.getElementsByClass("p-price").eq(0).text();
            String title = el.getElementsByClass("p-name").eq(0).text();
            Content content = new Content(title, img, price);
            goodList.add(content);
        }

        return goodList;

    }

    public static void main(String[] args) throws IOException {

        / / get request at https://search.jd.com/Search?keyword=java
        String url = "https://search.jd.com/Search?keyword=java";

        // Parse the web page (return Document is the browser Document object)
        Document document = Jsoup.parse(new URL(url), 30000);

        // All the methods available in JS can be used here
        Element element = document.getElementById("J_goodsList");
// System.out.println(element.html());

        // Get all the li elements
        Elements elements = element.getElementsByTag("li");
        // Get the contents of the element
        for (Element el : elements) {
            // For sites with lots of images, all images are loaded source-data-lazy-img
            String img = el.getElementsByTag("img").eq(0).attr("src");
            String price = el.getElementsByClass("p-price").eq(0).text();
            String title = el.getElementsByClass("p-name").eq(0).text();
            System.out.println("= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = ="); System.out.println(img); System.out.println(price); System.out.println(title); }}}Copy the code

Analyze jd search products

The business logic layer ContentService

/** * Business logic layer **@author mxz
 */
@Service
public class ContentService {

    public static final String ES_INDEX = "jd_goods";

    @Autowired
    @Qualifier("restHighLevelClient")
    private RestHighLevelClient client;

    /** * 1 parse data into es **@param keywords
     * @return
     * @throws Exception
     */
    public Boolean parseContent(String keywords) throws Exception {
        List<Content> contents = new HtmlParseUtil().parseJD(keywords);

        // Insert the queried data into es
        BulkRequest bulkRequest = new BulkRequest();
        bulkRequest.timeout(TimeValue.timeValueSeconds(2));
        for (int i = 0; i < contents.size(); i++) {

            bulkRequest.add(new IndexRequest(ES_INDEX)
                    .source(JSON.toJSONString(contents.get(i)), XContentType.JSON));

        }
        BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);

        return! bulkResponse.hasFailures(); }/** * 2 obtain the data to implement search function **@param keywords
     * @param pageNo
     * @param pageSize
     * @return
     * @throws IOException
     */
    public List<Map<String, Object>> searchPage(String keywords, int pageNo, int pageSize) throws IOException {
        if (pageNo <= 1) {
            pageNo = 1;
        }

        // Conditional search
        SearchRequest searchRequest = new SearchRequest(ES_INDEX);
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();

        / / paging
        searchSourceBuilder.from(pageNo);
        searchSourceBuilder.size(pageSize);

        // Precise matching
        TermQueryBuilder termQuery = QueryBuilders.termQuery("title", keywords);
        searchSourceBuilder.query(termQuery);
        searchSourceBuilder.timeout(TimeValue.timeValueSeconds(60));

        // Perform a search
        searchRequest.source(searchSourceBuilder);

        // Query through the client
        SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);

        // Parse the result
        List<Map<String, Object>> list = new ArrayList<>();
        for (SearchHit documentFields : searchResponse.getHits().getHits()) {
            list.add(documentFields.getSourceAsMap());
        }
        return list;
    }


    /** * 3 implement search function highlight **@param keywords
     * @param pageNo
     * @param pageSize
     * @return
     * @throws IOException
     */
    public List<Map<String, Object>> searchPageHighlighter(String keywords, int pageNo, int pageSize) throws IOException {

        if (pageNo <= 1) {
            pageNo = 1;
        }

        // Conditional search
        SearchRequest searchRequest = new SearchRequest(ES_INDEX);
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();

        / / paging
        searchSourceBuilder.from(pageNo);
        searchSourceBuilder.size(pageSize);

        // Precise matching
        TermQueryBuilder termQuery = QueryBuilders.termQuery("title", keywords);
        searchSourceBuilder.query(termQuery);
        searchSourceBuilder.timeout(TimeValue.timeValueSeconds(60));

        // Generate the highlighted query
        HighlightBuilder highlightBuilder = new HighlightBuilder();
        // Highlight query fields
        highlightBuilder.field("title");
        // If more than one field is highlighted, this must be false
        highlightBuilder.requireFieldMatch(false);
        // Highlight Settings
        highlightBuilder.preTags("<span style = 'color:red'>");
        highlightBuilder.postTags("</span>");
        // The following two items must be configured if you want to highlight fields with a lot of words, such as text content, otherwise they will result in incomplete highlighting, missing content, etc
        // The maximum number of highlighted fragments
        highlightBuilder.fragmentSize(800000);
        // Get the highlighted fragment from the first fragment
        highlightBuilder.numOfFragments(0);
        searchSourceBuilder.highlighter(highlightBuilder);


        // Perform a search
        searchRequest.source(searchSourceBuilder);

        // Query through the client
        SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);

        // Parse the result
        List<Map<String, Object>> list = new ArrayList<>();
        for (SearchHit documentFields : searchResponse.getHits().getHits()) {
            // Parse the highlighted field
            Map<String, HighlightField> highlightFields = documentFields.getHighlightFields();
            HighlightField title = highlightFields.get("title");
            Map<String, Object> sourceAsMap = documentFields.getSourceAsMap();  // The original result
            // Parse highlighted fields to replace the original fields with highlighted fields
            // Make sure to check if it is null, or the first result you match is not highlighted, and you will get a null pointer exception. This error takes a long time to start
            if(title ! =null) {
                Text[] fragments = title.fragments();
                String newTitle = "";
                for (Text fragment : fragments) {
                    newTitle += fragment;
                }
                sourceAsMap.put("title", newTitle);
                System.out.println(newTitle);
            }
            list.add(sourceAsMap);
        }
        returnlist; }}Copy the code

Control layer RestController

@RestController
public class ContentController {

    @Autowired
    private ContentService contentService;

    @GetMapping("/parse/{keywords}")
    public Boolean parse(@PathVariable("keywords") String keywords) throws Exception {
        return contentService.parseContent(keywords);
    }

    @GetMapping("/search/{keywords}/{pageNo}/{pageSize}")
    public List<Map<String, Object>> searchPage(@PathVariable("keywords") String keywords,
                                                @PathVariable("pageNo") int pageNo,
                                                @PathVariable("pageSize") int pageSize) throws IOException {
        // return contentService.searchPage(keywords, pageNo, pageSize);
        / / highlight
        returncontentService.searchPageHighlighter(keywords, pageNo, pageSize); }}Copy the code

The front page is myindex.html

<! DOCTYPEhtml>
<html lang="zh-en" xmlns:th="http://www.thymeleaf.org">
<head>
    <meta charset="UTF-8">
    <title>Full stack self-taught community search</title>

    <link rel="stylesheet" href="https://v4.bootcss.com/docs/4.6/dist/css/bootstrap.min.css">
    <link rel="apple-touch-icon" href="https://v4.bootcss.com/docs/4.6/assets/img/favicons/apple-touch-icon.png"
          sizes="180x180">
    <link rel="icon" href="https://v4.bootcss.com/docs/4.6/assets/img/favicons/favicon-32x32.png" sizes="32x32"
          type="image/png">
    <link rel="icon" href="https://v4.bootcss.com/docs/4.6/assets/img/favicons/favicon-16x16.png" sizes="16x16"
          type="image/png">
    <link rel="mask-icon" href="https://v4.bootcss.com/docs/4.6/assets/img/favicons/safari-pinned-tab.svg"
          color="#563d7c">
    <link rel="icon" href="https://v4.bootcss.com/docs/4.6/assets/img/favicons/favicon.ico">
    <meta name="msapplication-config" content="/ docs / 4.6 / assets/img/favicons/browserconfig XML">
    <meta name="theme-color" content="#563d7c">

    <style>
        .bd-placeholder-img {
            font-size: 1.125 rem;
            text-anchor: middle;
            -webkit-user-select: none;
            -moz-user-select: none;
            -ms-user-select: none;
            user-select: none;
        }

        @media (min-width: 768px) {
            .bd-placeholder-img-lg {
                font-size: 3.5 rem; }}</style>
</head>
<body>
<div id="app">
    <nav class="navbar navbar-expand-md navbar-dark bg-dark mb-4">
        <a class="navbar-brand" href="https://v4.bootcss.com/docs/examples/navbar-static/#">Full stack self-learning community</a>
        <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarCollapse"
                aria-controls="navbarCollapse" aria-expanded="false" aria-label="Toggle navigation">
            <span class="navbar-toggler-icon"></span>
        </button>
        <div class="collapse navbar-collapse" id="navbarCollapse">
            <ul class="navbar-nav mr-auto">
                <li class="nav-item active">
                    <a class="nav-link" href="https://v4.bootcss.com/docs/examples/navbar-static/#">The home page<span
                            class="sr-only">(current)</span></a>
                </li>
                <li class="nav-item">
                    <a class="nav-link" href="https://v4.bootcss.com/docs/examples/navbar-static/#">Friends of the chain</a>
                </li>
                <li class="nav-item">
                    <a class="nav-link disabled" href="https://v4.bootcss.com/docs/examples/navbar-static/#"
                       tabindex="1" aria-disabled="true">The control panel</a>
                </li>
            </ul>
            <form class="form-inline mt-2 mt-md-0">
                <input class="form-control mr-sm-2" v-model="keyword" type="text" placeholder="Enter name"
                       aria-label="Search">
                <button class="btn btn-outline-success my-2 my-sm-0" @click.prevent="searchKey" type="submit">search</button>
            </form>
        </div>
    </nav>

    <main role="main" class="container">
        <div class="row">
            <div class="col-md-4" v-for="result in results">
                <img :src="result.img" class="img-thumbnail"/>
                <p><a v-html="result.title"></a></p>
                <p class="lead">{{result.price}}</p>
            </div>
        </div>
    </main>
</div>
<script type="text/javascript" th:src="@{/js/axios.min.js}"></script>
<script type="text/javascript" th:src="@{/js/vue.min.js}"></script>
<script>
    new Vue({
        el: '#app'.data: {
            keyword: ' '.// Search for keywords
            results: []},methods: {
            searchKey() {
                var keyword = this.keyword;
                console.log(keyword);
                // Connect to the back-end interface
                axios.get('/search/' + keyword + '/ 1 /' + '10').then(response= > {
                    console.log(response);
                    this.results = response.data;  // Bind the data}}}})</script>

</body>
</html>
Copy the code

Project running

• Before the search

• After the search

This article has been uploaded to gitee gitee.com/codingce/he… Project address: github.com/xzMhehe/cod…