GoGreen PC Tune-Up™
Learn More

Insta-Install™
this is how ssl encrypt our websites
MTNCOMP | List View | Table View | myBlog (1772 Entries)
myBlog Home

Blog


C# and SQL internet search engine database starter kit

SQL

by Mountain Computers Inc., Publication Date: Friday, August 2, 2019

View Count: 1607, Keywords: C#, Website, Content, Processing, Hashtags: #C# #Website #Content #Processing



this is some crazy ass code for processing domain website content, its not perfect, but it is a brute force process and works well with my other tools and calls to this app... 
 
the batch caller is here (just change your '10' to whatever number of launches you want and the static ip to the sql server as well... can't show you the DB but its 180mil records and more...the resultant db content is over 8tb.
 
--- batch file
 

del "C:\ProgramData\ProjectSearch43\ProjectSearch43\1.0.0.0\quit.txt"
for /l %%x in (1, 1, 10) do (
    echo %%x
    start projectsearch43 ip 192.168.0.200 process
)
 
the C# code is here.. 
-- projectsearch43.exe
 
Imports System.IO
Imports System.Data.SqlClient
Imports System.Threading
Public Class Form1
    Inherits System.Windows.Forms.Form
    'Create ADO.NET objects.
    Private myConn As SqlConnection
    Private myCmd As SqlCommand
    Private myReader As SqlDataReader
    Private results As String
    Private maxDBrecords As Integer
    Private DbDNS As String
    Private DbDNSTable As String
    Private DataSourceIPAddress As String
    Private CloseAppBit As Integer

    Private Sub BtnExit_Click(sender As Object, e As EventArgs) Handles BtnExit.Click
        Application.Exit()
    End Sub

    Private Sub Form1_Load(sender As Object, e As EventArgs) Handles MyBase.Load
        Debug.Print("Form1_Load")

        RichTextBox1.Clear()
        CbDNSdb.Items.Add("DNSRecords1")
        CbDNSdb.Items.Add("DNSRecords2")
        CbDNSdb.Items.Add("DNSRecords3")
        CbDNSdb.Items.Add("DNSRecords4")
        CbDNSdb.Items.Add("DNSRecords5")
        CbServerIP.Items.Add("RG4S-PC")
        CbServerIP.Items.Add("127.0.0.1")
        TbTable.Text = "DNSRecords2"
        TbIP.Text = "127.0.0.1"
        DataSourceIPAddress = TbIP.Text
        DbDNSTable = TbTable.Text
        DbDNS = "dbDNS"
        CloseAppBit = 0
        ListStatus.Items.Add("App loaded.")

        Debug.Print("# of Args: " & My.Application.CommandLineArgs.Count)
        Dim Args
        For Args = 0 To My.Application.CommandLineArgs.Count - 1
            Debug.Print(My.Application.CommandLineArgs(Args))
            ListStatus.Items.Add("* " & My.Application.CommandLineArgs(Args))
            Select Case My.Application.CommandLineArgs(Args)
                Case "ip"
                    TbIP.Text = My.Application.CommandLineArgs(Args + 1)
                    DataSourceIPAddress = TbIP.Text
                Case "process"
                    Me.Show()
                    Call ProcessDB()
                Case Else

            End Select
        Next
        Me.Refresh()
    End Sub

    Private Sub ConnectDB(myquery, numparams)
        ListStatus.Items.Add("Connecting db")
        'Create a Connection object.
        myConn = New SqlConnection("Initial Catalog=" & DbDNS & ";" & "Data Source=tcp:" & DataSourceIPAddress & ";User ID=sa;Password=yoursapasswd")
        'Create a Command object.
        myCmd = myConn.CreateCommand
        'myCmd.CommandText = "SELECT max(row_id) FROM DNSRecords1"
        myCmd.CommandText = myquery

        ListStatus.Items.Add("Opening db")
        'Open the connection.
        myConn.Open()

        results = ""
        myReader = myCmd.ExecuteReader()

        ListStatus.Items.Add("Reading db")
        'Concatenate the query result into a string.
        Do While myReader.Read()
            Select Case numparams

                Case 0
                    'don't return anything, just execute query
                Case 1
                    results = results & myReader(0) & vbCrLf
                Case 2
                    results = results & myReader(0) & vbTab & myReader(1) & vbCrLf
                Case 3
                    results = results & myReader(0) & vbTab & myReader(1) & vbTab & myReader(2) & vbCrLf
                Case 4
                    results = results & myReader(0) & vbTab & myReader(1) & vbTab & myReader(2) & vbTab & myReader(3) & vbCrLf
                Case 5
                    results = results & myReader(0) & vbTab & myReader(1) & vbTab & myReader(2) & vbTab & myReader(3) & vbTab & myReader(4) & vbCrLf

                Case Else

            End Select
            If numparams > 1 Then
                ListResults.Items.Add(Now() & ":" & results)
                results = ""
            End If

        Loop
        'Display results.
        'MsgBox(results)
        ListResults.Items.Add(Now() & ":" & results)
        'Close the reader and the database connection.
        myReader.Close()
        ListStatus.Items.Add("Clozing db")
        myConn.Close()

    End Sub

    Private Sub BtnSearch_Click(sender As Object, e As EventArgs) Handles BtnSearch.Click
        If iStart.Text <> "" And iEnd.Text <> "" Or zMyQuery.Text <> "" Then
            Debug.Print("good. start and end or query")
            If zMyQuery.Text <> "" Then
                Call ConnectDB(zMyQuery.Text, 1)
            Else
                Dim s
                s = "select * from " & TbTable.Text & " WHERE row_id >= " & iStart.Text & " and row_id <= " & iEnd.Text
                Call ConnectDB(s, 5)
            End If
        Else
            ListResults.Items.Add("no good. nothign entered to find.")
            Debug.Print("no good. nothing entered to find.")
        End If

    End Sub

    Private Sub BtnClear_Click(sender As Object, e As EventArgs) Handles BtnClear.Click
        ListResults.Items.Clear()
    End Sub

    Private Sub BtnProcessDB_Click(sender As Object, e As EventArgs) Handles BtnProcessDB.Click
        DataSourceIPAddress = TbIP.Text
        Call ClearQuitFlags()
        Call ProcessDB()
    End Sub
    Private Sub ProcessDB()
        'process the database
        Dim x, y, d
        x = 0   'hits
        y = 0   'tries, misses
        d = "" 'domain name
        If ChkBoxStopProcessing.Checked Then
            ChkBoxStopProcessing.Checked = False
        End If
        LblStartDateTime.Text = Now()
        Dim s
        s = "select max(row_id) from " & TbTable.Text
        Call ConnectDB(s, 1)
        maxDBrecords = results
        Debug.Print("maxrecords=" & maxDBrecords)
        ListStatus.Items.Add(Now() & " r=" & maxDBrecords)
        'myConn = New SqlConnection("Initial Catalog=dbDNS;Data Source=localhost;User ID=sa;Password=yoursapasswd")
        myConn = New SqlConnection("Initial Catalog=" & DbDNS & ";Data Source=tcp:" & DataSourceIPAddress & ";User ID=sa;Password=yoursapasswd")
        ListStatus.Items.Add("Opening db")
        myConn.Open()
        ListStatus.Items.Add("Reading db")

        Dim r
        Do Until ChkBoxStopProcessing.CheckState = CheckState.Checked
            'Application.DoEvents()
            'r = CInt(Math.Ceiling(Rnd() * maxDBrecords) + 1)
            'Call CheckQuitApp()
            Static Generator As System.Random = New System.Random()
            r = Generator.Next(1, maxDBrecords)
            'as long as r is within bounds
            If r > 0 And r < maxDBrecords Then
                'Application.DoEvents()
                Debug.Print("r = " & r)
                myCmd = myConn.CreateCommand
                s = "select domain_name from " & TbTable.Text & " WHERE substring(recordtype,9,5) = 'cname' and been_processed is null and row_id = " & r
                myCmd.CommandText = s
                myReader = myCmd.ExecuteReader()
                Do While myReader.Read()
                    results = results & myReader(0) & vbCrLf
                    'ListResults.Items.Add("[" & x & "]" & r & "," & results)
                    'x = x + 1
                    'LblMatches.Text = x.ToString
                    d = Mid(results, 9, Len(results) - 11)
                    Debug.Print("domainame = " & results & " - " & d)
                    'the cname / domain has www. in it even better, ignore everything else
                    'If InStr(d, ".com") > 0 And InStr(d, "www.") > 0 And Len(d) < 32 Then
                    If Microsoft.VisualBasic.Right(d, 4) = ".com" And InStr(d, "www.") > 0 And Len(d) < 32 Then
                        x = x + 1
                        LblMatches.Text = x.ToString
                        ListResults.Items.Add("[" & Now & ":" & x & "]" & r & "," & results)
                        Application.DoEvents()
                        Call GetWebText(r, d)
                    End If
                Loop
                results = ""
                y = y + 1
                LblTries.Text = y.ToString
                myReader.Close()
                Call CheckQuitApp()
                If x Mod 20 = 0 Then
                    ListResults.Items.Clear()
                    'ListStatus.Items.Clear()
                End If
            End If
            'from quit.txt found
            If CloseAppBit = 1 Then
                myConn.Close()
                Application.Exit()
            End If
        Loop

        ListStatus.Items.Add("Klozing db.")
        myConn.Close()
        'analyze and score the records
        'repeat in this loop, and pause (remember there are other workers involved)
    End Sub
    Private Sub GetWebText(r As Integer, v As String)
        Dim myConn2 As SqlConnection
        Dim myCmd2 As SqlCommand
        Dim s
        Try
            Application.DoEvents()
            Dim sourceString As String = New System.Net.WebClient().DownloadString("http://" & v)
            'myConn2 = New SqlConnection("Initial Catalog=dbDNS;Data Source=localhost;User ID=sa;Password=yoursapasswd")
            myConn2 = New SqlConnection("Initial Catalog=" & DbDNS & ";Data Source=tcp:" & DataSourceIPAddress & ";User ID=sa;Password=yoursapasswd")
            myConn2.Open()
            myCmd2 = myConn2.CreateCommand
            s = "UPDATE " & TbTable.Text & " SET processed_datetime = getdate(), been_processed = 1, titlepos = " & InStr(sourceString, " ") & ",homepagetext = '" & GetSQLText(sourceString) & "' WHERE row_id = " & r<br />             myCmd2.CommandText = s<br />             myCmd2.ExecuteNonQuery()<br />             myConn2.Close()<br />             sourceString = ""<br />         Catch ex As Exception<br />             Debug.Print("error" & ex.Message & " trying to reach " & v)<br />         Finally<br />             'do nothing<br />         End Try<br /> <br />     End Sub<br /> <br />     Private Function GetSQLText(sText)<br />         Dim bEnd<br />         Dim i<br />         Dim iLast<br />         Dim sSQLText<br />         sSQLText = ""<br />         bEnd = False<br />         i = 1<br />         iLast = 1<br />         While Not bEnd<br />             i = InStr(iLast, sText, "'")<br />             If i > 0 Then<br />                 sSQLText = sSQLText & Mid(sText, iLast, i - iLast) & "''"<br />                 iLast = i + 1<br />             Else<br />                 sSQLText = sSQLText & Mid(sText, iLast)<br />                 bEnd = True<br />             End If<br />         End While<br />         GetSQLText = sSQLText<br />     End Function<br /> <br />     Private Sub CbDNSdb_SelectedIndexChanged(sender As Object, e As EventArgs) Handles CbDNSdb.SelectedIndexChanged<br />         'change db table<br />         TbTable.Text = CbDNSdb.SelectedItem<br />     End Sub<br /> <br />     Private Sub CbServerIP_SelectedIndexChanged(sender As Object, e As EventArgs) Handles CbServerIP.SelectedIndexChanged<br />         'change server ip<br />         TbIP.Text = CbServerIP.SelectedItem<br />     End Sub<br /> <br />     Private Sub BtnShowStats_Click(sender As Object, e As EventArgs) Handles BtnShowStats.Click<br />         'show the stats in the richtextbox1<br />         DataSourceIPAddress = TbIP.Text<br />         Dim s, txt<br />         s = ""<br />         txt = ""<br />         RichTextBox1.Text = ""<br />         Try<br />             s = "select max(row_id) from " & TbTable.Text<br />             Call ConnectDB(s, 1)<br />             txt = "Max Records: " & results<br />             s = "select count(*) from " & TbTable.Text & " WHERE been_processed = 1"<br />             Call ConnectDB(s, 1)<br />             txt = txt & "Processed: " & results & "Date/Time: " & Now()<br />             RichTextBox1.Text = txt<br />         Catch ex As Exception<br />             Debug.Print("error" & ex.Message & " ,getting stats error. ")<br />         Finally<br />             'do nothing<br />         End Try<br />     End Sub<br /> <br />     Private Sub BtnBetaTest1_Click(sender As Object, e As EventArgs) Handles BtnBetaTest1.Click<br />         'get a webpage, naming my homepage<br />         Dim s<br />         s = "https://mountaincomputers.org"<br />         s = InputBox("enter domain", "get domain page", s)<br />         Debug.Print("get my home page")<br />         'Dim sourceString As String = New System.Net.WebClient().DownloadString("https://mountaincomputers.org")<br />         Dim sourceString As String = New System.Net.WebClient().DownloadString(s)<br />         'Debug.Print(sourceString)<br />         RichTextBox1.Text = ""<br />         RichTextBox1.Text = sourceString<br />     End Sub<br /> <br />     Private Sub BtnBetaTest2_Click(sender As Object, e As EventArgs) Handles BtnBetaTest2.Click<br />         'analyze text to determine ..<br />         '1. whereas domain and title equivalenc<br />         '2. title Byte distance from top of file<br />         '2a. length of the file? how big? number of line feeds character returns? lines > 1024?<br />         '3. existence of domain to title, to keywords, to description<br />         '3a. viewport meta tag and what value?<br />         '3b. <br /> <br />         Dim s, x<br />         s = ""<br />         x = 0<br />         Try<br />             ListStatus.Items.Add("Konnecting 2 db.")<br />             'Create a Connection object.<br />             myConn = New SqlConnection("Initial Catalog=" & DbDNS & ";" & "Data Source=tcp:" & DataSourceIPAddress & ";User ID=sa;Password=yoursapasswd")<br />             'Create a Command object.<br />             myCmd = myConn.CreateCommand<br />             'myCmd.CommandText = "SELECT max(row_id) FROM DNSRecords1"<br />             s = "SELECT row_id, homepagetext from " & TbTable.Text & " WHERE been_processed = 1 and titlepos > 1024 ORDER BY row_id"<br />             myCmd.CommandText = s<br /> <br />             ListStatus.Items.Add("Opening db.")<br />             'Open the connection.<br />             myConn.Open()<br />             Application.DoEvents()<br />             x = 0<br />             results = ""<br />             myReader = myCmd.ExecuteReader()<br />             ListStatus.Items.Add("Reading db.")<br />             'Concatenate the query result into a string.<br />             'Do While myReader.Read() And x < 20<br />             Do While myReader.Read()<br />                 Me.Refresh()<br />                 Application.DoEvents()<br />                 results = x & ", row_id: " & myReader(0) & "," & InStr(myReader(1), "<title>")<br />                 ListResults.Items.Add(results)<br />                 'Application.DoEvents()<br />                 results = ""<br />                 x = x + 1<br />                 Debug.Print(x)<br />             Loop<br />             'Close the reader and the database connection.<br />             Debug.Print(x)<br />             myReader.Close()<br />             ListStatus.Items.Add("Klozing db.")<br />             myConn.Close()<br />         Catch ex As Exception<br />             Debug.Print("error" & ex.Message & ".")<br />         Finally<br />             'do nothing<br />         End Try<br />         Debug.Print("done")<br />     End Sub<br /> <br />     Private Sub BtnBetaTest3_Click(sender As Object, e As EventArgs) Handles BtnBetaTest3.Click<br /> <br /> <br />         Try<br /> <br />             Dim path1 As String = Application.CommonAppDataPath & "\" & "quit.txt"<br /> <br />             Dim fi As FileInfo = New FileInfo(path1)<br /> <br />             'Create a file to write to.<br />             Dim sw As StreamWriter = fi.CreateText()<br /> <br />             sw.WriteLine("quit")<br />             sw.Flush()<br />             sw.Close()<br /> <br />         Catch ex As Exception<br />             'Debug.Print("Error saving current game file")<br /> <br />         Finally<br />         End Try<br /> <br />         Call CheckQuitApp()<br />     End Sub<br />     Private Sub CheckQuitApp()<br />         Try<br />             Debug.Print(Application.CommonAppDataPath & "\" & "quit.txt")<br />             If FileLen(Application.CommonAppDataPath & "\" & "quit.txt") > 0 Then<br />                 'Application.Exit()<br />                 ChkBoxStopProcessing.Checked = True<br />                 CloseAppBit = 1<br />             End If<br />         Catch<br />         End Try<br /> <br />     End Sub<br />     Private Sub ClearQuitFlags()<br />         Try<br />             Debug.Print(Application.CommonAppDataPath & "\" & "quit.txt")<br />             If FileLen(Application.CommonAppDataPath & "\" & "quit.txt") > 0 Then<br />                 'Application.Exit()<br />                 My.Computer.FileSystem.DeleteFile(Application.CommonAppDataPath & "\" & "quit.txt")<br />                 CloseAppBit = 0<br />             End If<br />         Catch<br />         End Try<br /> <br />     End Sub<br /> <br /> <br />     Private Sub BtnStopEveryone_Click(sender As Object, e As EventArgs) Handles BtnStopEveryone.Click<br /> <br />         Try<br /> <br />             Dim path1 As String = Application.CommonAppDataPath & "\" & "quit.txt"<br /> <br />             Dim fi As FileInfo = New FileInfo(path1)<br /> <br />             'Create a file to write to.<br />             Dim sw As StreamWriter = fi.CreateText()<br /> <br />             sw.WriteLine("quit")<br />             sw.Flush()<br />             sw.Close()<br /> <br />         Catch ex As Exception<br />             'Debug.Print("Error saving current game file")<br /> <br />         Finally<br />         End Try<br /> <br />     End Sub<br /> End Class<br />  </div> <div> </div> <div> </div> <div> </div></body></html>

if you found this article helpful, consider contributing $10, 20 an Andrew Jackson or so..to the author. more authors coming soon
FYI we use paypal or patreon, patreon has 3x the transaction fees, so we don't, not yet.

© 2024 myBlog™ v1.1 All rights reserved. We count views as reads, so let's not over think it.