C# and SQL internet search engine database starter kit
by Mountain Computers Inc., Publication Date: Friday, August 2, 2019
View Count: 1607, Keywords: C#, Website, Content, Processing, Hashtags: #C# #Website #Content #Processing
this is some crazy ass code for processing domain website content, its not perfect, but it is a brute force process and works well with my other tools and calls to this app...
the batch caller is here (just change your '10' to whatever number of launches you want and the static ip to the sql server as well... can't show you the DB but its 180mil records and more...the resultant db content is over 8tb.
--- batch file
del "C:\ProgramData\ProjectSearch43\ProjectSearch43\1.0.0.0\quit.txt"
for /l %%x in (1, 1, 10) do (
echo %%x
start projectsearch43 ip 192.168.0.200 process
)
the C# code is here..
-- projectsearch43.exe
Imports System.IO
Imports System.Data.SqlClient
Imports System.Threading
Public Class Form1
Inherits System.Windows.Forms.Form
'Create ADO.NET objects.
Private myConn As SqlConnection
Private myCmd As SqlCommand
Private myReader As SqlDataReader
Private results As String
Private maxDBrecords As Integer
Private DbDNS As String
Private DbDNSTable As String
Private DataSourceIPAddress As String
Private CloseAppBit As Integer
Private Sub BtnExit_Click(sender As Object, e As EventArgs) Handles BtnExit.Click
Application.Exit()
End Sub
Private Sub Form1_Load(sender As Object, e As EventArgs) Handles MyBase.Load
Debug.Print("Form1_Load")
RichTextBox1.Clear()
CbDNSdb.Items.Add("DNSRecords1")
CbDNSdb.Items.Add("DNSRecords2")
CbDNSdb.Items.Add("DNSRecords3")
CbDNSdb.Items.Add("DNSRecords4")
CbDNSdb.Items.Add("DNSRecords5")
CbServerIP.Items.Add("RG4S-PC")
CbServerIP.Items.Add("127.0.0.1")
TbTable.Text = "DNSRecords2"
TbIP.Text = "127.0.0.1"
DataSourceIPAddress = TbIP.Text
DbDNSTable = TbTable.Text
DbDNS = "dbDNS"
CloseAppBit = 0
ListStatus.Items.Add("App loaded.")
Debug.Print("# of Args: " & My.Application.CommandLineArgs.Count)
Dim Args
For Args = 0 To My.Application.CommandLineArgs.Count - 1
Debug.Print(My.Application.CommandLineArgs(Args))
ListStatus.Items.Add("* " & My.Application.CommandLineArgs(Args))
Select Case My.Application.CommandLineArgs(Args)
Case "ip"
TbIP.Text = My.Application.CommandLineArgs(Args + 1)
DataSourceIPAddress = TbIP.Text
Case "process"
Me.Show()
Call ProcessDB()
Case Else
End Select
Next
Me.Refresh()
End Sub
Private Sub ConnectDB(myquery, numparams)
ListStatus.Items.Add("Connecting db")
'Create a Connection object.
myConn = New SqlConnection("Initial Catalog=" & DbDNS & ";" & "Data Source=tcp:" & DataSourceIPAddress & ";User ID=sa;Password=yoursapasswd")
'Create a Command object.
myCmd = myConn.CreateCommand
'myCmd.CommandText = "SELECT max(row_id) FROM DNSRecords1"
myCmd.CommandText = myquery
ListStatus.Items.Add("Opening db")
'Open the connection.
myConn.Open()
results = ""
myReader = myCmd.ExecuteReader()
ListStatus.Items.Add("Reading db")
'Concatenate the query result into a string.
Do While myReader.Read()
Select Case numparams
Case 0
'don't return anything, just execute query
Case 1
results = results & myReader(0) & vbCrLf
Case 2
results = results & myReader(0) & vbTab & myReader(1) & vbCrLf
Case 3
results = results & myReader(0) & vbTab & myReader(1) & vbTab & myReader(2) & vbCrLf
Case 4
results = results & myReader(0) & vbTab & myReader(1) & vbTab & myReader(2) & vbTab & myReader(3) & vbCrLf
Case 5
results = results & myReader(0) & vbTab & myReader(1) & vbTab & myReader(2) & vbTab & myReader(3) & vbTab & myReader(4) & vbCrLf
Case Else
End Select
If numparams > 1 Then
ListResults.Items.Add(Now() & ":" & results)
results = ""
End If
Loop
'Display results.
'MsgBox(results)
ListResults.Items.Add(Now() & ":" & results)
'Close the reader and the database connection.
myReader.Close()
ListStatus.Items.Add("Clozing db")
myConn.Close()
End Sub
Private Sub BtnSearch_Click(sender As Object, e As EventArgs) Handles BtnSearch.Click
If iStart.Text <> "" And iEnd.Text <> "" Or zMyQuery.Text <> "" Then
Debug.Print("good. start and end or query")
If zMyQuery.Text <> "" Then
Call ConnectDB(zMyQuery.Text, 1)
Else
Dim s
s = "select * from " & TbTable.Text & " WHERE row_id >= " & iStart.Text & " and row_id <= " & iEnd.Text
Call ConnectDB(s, 5)
End If
Else
ListResults.Items.Add("no good. nothign entered to find.")
Debug.Print("no good. nothing entered to find.")
End If
End Sub
Private Sub BtnClear_Click(sender As Object, e As EventArgs) Handles BtnClear.Click
ListResults.Items.Clear()
End Sub
Private Sub BtnProcessDB_Click(sender As Object, e As EventArgs) Handles BtnProcessDB.Click
DataSourceIPAddress = TbIP.Text
Call ClearQuitFlags()
Call ProcessDB()
End Sub
Private Sub ProcessDB()
'process the database
Dim x, y, d
x = 0 'hits
y = 0 'tries, misses
d = "" 'domain name
If ChkBoxStopProcessing.Checked Then
ChkBoxStopProcessing.Checked = False
End If
LblStartDateTime.Text = Now()
Dim s
s = "select max(row_id) from " & TbTable.Text
Call ConnectDB(s, 1)
maxDBrecords = results
Debug.Print("maxrecords=" & maxDBrecords)
ListStatus.Items.Add(Now() & " r=" & maxDBrecords)
'myConn = New SqlConnection("Initial Catalog=dbDNS;Data Source=localhost;User ID=sa;Password=yoursapasswd")
myConn = New SqlConnection("Initial Catalog=" & DbDNS & ";Data Source=tcp:" & DataSourceIPAddress & ";User ID=sa;Password=yoursapasswd")
ListStatus.Items.Add("Opening db")
myConn.Open()
ListStatus.Items.Add("Reading db")
Dim r
Do Until ChkBoxStopProcessing.CheckState = CheckState.Checked
'Application.DoEvents()
'r = CInt(Math.Ceiling(Rnd() * maxDBrecords) + 1)
'Call CheckQuitApp()
Static Generator As System.Random = New System.Random()
r = Generator.Next(1, maxDBrecords)
'as long as r is within bounds
If r > 0 And r < maxDBrecords Then
'Application.DoEvents()
Debug.Print("r = " & r)
myCmd = myConn.CreateCommand
s = "select domain_name from " & TbTable.Text & " WHERE substring(recordtype,9,5) = 'cname' and been_processed is null and row_id = " & r
myCmd.CommandText = s
myReader = myCmd.ExecuteReader()
Do While myReader.Read()
results = results & myReader(0) & vbCrLf
'ListResults.Items.Add("[" & x & "]" & r & "," & results)
'x = x + 1
'LblMatches.Text = x.ToString
d = Mid(results, 9, Len(results) - 11)
Debug.Print("domainame = " & results & " - " & d)
'the cname / domain has www. in it even better, ignore everything else
'If InStr(d, ".com") > 0 And InStr(d, "www.") > 0 And Len(d) < 32 Then
If Microsoft.VisualBasic.Right(d, 4) = ".com" And InStr(d, "www.") > 0 And Len(d) < 32 Then
x = x + 1
LblMatches.Text = x.ToString
ListResults.Items.Add("[" & Now & ":" & x & "]" & r & "," & results)
Application.DoEvents()
Call GetWebText(r, d)
End If
Loop
results = ""
y = y + 1
LblTries.Text = y.ToString
myReader.Close()
Call CheckQuitApp()
If x Mod 20 = 0 Then
ListResults.Items.Clear()
'ListStatus.Items.Clear()
End If
End If
'from quit.txt found
If CloseAppBit = 1 Then
myConn.Close()
Application.Exit()
End If
Loop
ListStatus.Items.Add("Klozing db.")
myConn.Close()
'analyze and score the records
'repeat in this loop, and pause (remember there are other workers involved)
End Sub
Private Sub GetWebText(r As Integer, v As String)
Dim myConn2 As SqlConnection
Dim myCmd2 As SqlCommand
Dim s
Try
Application.DoEvents()
Dim sourceString As String = New System.Net.WebClient().DownloadString("http://" & v)
'myConn2 = New SqlConnection("Initial Catalog=dbDNS;Data Source=localhost;User ID=sa;Password=yoursapasswd")
myConn2 = New SqlConnection("Initial Catalog=" & DbDNS & ";Data Source=tcp:" & DataSourceIPAddress & ";User ID=sa;Password=yoursapasswd")
myConn2.Open()
myCmd2 = myConn2.CreateCommand
s = "UPDATE " & TbTable.Text & " SET processed_datetime = getdate(), been_processed = 1, titlepos = " & InStr(sourceString, "
") & ",homepagetext = '" & GetSQLText(sourceString) & "' WHERE row_id = " & r<br />
myCmd2.CommandText = s<br />
myCmd2.ExecuteNonQuery()<br />
myConn2.Close()<br />
sourceString = ""<br />
Catch ex As Exception<br />
Debug.Print("error" & ex.Message & " trying to reach " & v)<br />
Finally<br />
'do nothing<br />
End Try<br />
<br />
End Sub<br />
<br />
Private Function GetSQLText(sText)<br />
Dim bEnd<br />
Dim i<br />
Dim iLast<br />
Dim sSQLText<br />
sSQLText = ""<br />
bEnd = False<br />
i = 1<br />
iLast = 1<br />
While Not bEnd<br />
i = InStr(iLast, sText, "'")<br />
If i > 0 Then<br />
sSQLText = sSQLText & Mid(sText, iLast, i - iLast) & "''"<br />
iLast = i + 1<br />
Else<br />
sSQLText = sSQLText & Mid(sText, iLast)<br />
bEnd = True<br />
End If<br />
End While<br />
GetSQLText = sSQLText<br />
End Function<br />
<br />
Private Sub CbDNSdb_SelectedIndexChanged(sender As Object, e As EventArgs) Handles CbDNSdb.SelectedIndexChanged<br />
'change db table<br />
TbTable.Text = CbDNSdb.SelectedItem<br />
End Sub<br />
<br />
Private Sub CbServerIP_SelectedIndexChanged(sender As Object, e As EventArgs) Handles CbServerIP.SelectedIndexChanged<br />
'change server ip<br />
TbIP.Text = CbServerIP.SelectedItem<br />
End Sub<br />
<br />
Private Sub BtnShowStats_Click(sender As Object, e As EventArgs) Handles BtnShowStats.Click<br />
'show the stats in the richtextbox1<br />
DataSourceIPAddress = TbIP.Text<br />
Dim s, txt<br />
s = ""<br />
txt = ""<br />
RichTextBox1.Text = ""<br />
Try<br />
s = "select max(row_id) from " & TbTable.Text<br />
Call ConnectDB(s, 1)<br />
txt = "Max Records: " & results<br />
s = "select count(*) from " & TbTable.Text & " WHERE been_processed = 1"<br />
Call ConnectDB(s, 1)<br />
txt = txt & "Processed: " & results & "Date/Time: " & Now()<br />
RichTextBox1.Text = txt<br />
Catch ex As Exception<br />
Debug.Print("error" & ex.Message & " ,getting stats error. ")<br />
Finally<br />
'do nothing<br />
End Try<br />
End Sub<br />
<br />
Private Sub BtnBetaTest1_Click(sender As Object, e As EventArgs) Handles BtnBetaTest1.Click<br />
'get a webpage, naming my homepage<br />
Dim s<br />
s = "https://mountaincomputers.org"<br />
s = InputBox("enter domain", "get domain page", s)<br />
Debug.Print("get my home page")<br />
'Dim sourceString As String = New System.Net.WebClient().DownloadString("https://mountaincomputers.org")<br />
Dim sourceString As String = New System.Net.WebClient().DownloadString(s)<br />
'Debug.Print(sourceString)<br />
RichTextBox1.Text = ""<br />
RichTextBox1.Text = sourceString<br />
End Sub<br />
<br />
Private Sub BtnBetaTest2_Click(sender As Object, e As EventArgs) Handles BtnBetaTest2.Click<br />
'analyze text to determine ..<br />
'1. whereas domain and title equivalenc<br />
'2. title Byte distance from top of file<br />
'2a. length of the file? how big? number of line feeds character returns? lines > 1024?<br />
'3. existence of domain to title, to keywords, to description<br />
'3a. viewport meta tag and what value?<br />
'3b. <br />
<br />
Dim s, x<br />
s = ""<br />
x = 0<br />
Try<br />
ListStatus.Items.Add("Konnecting 2 db.")<br />
'Create a Connection object.<br />
myConn = New SqlConnection("Initial Catalog=" & DbDNS & ";" & "Data Source=tcp:" & DataSourceIPAddress & ";User ID=sa;Password=yoursapasswd")<br />
'Create a Command object.<br />
myCmd = myConn.CreateCommand<br />
'myCmd.CommandText = "SELECT max(row_id) FROM DNSRecords1"<br />
s = "SELECT row_id, homepagetext from " & TbTable.Text & " WHERE been_processed = 1 and titlepos > 1024 ORDER BY row_id"<br />
myCmd.CommandText = s<br />
<br />
ListStatus.Items.Add("Opening db.")<br />
'Open the connection.<br />
myConn.Open()<br />
Application.DoEvents()<br />
x = 0<br />
results = ""<br />
myReader = myCmd.ExecuteReader()<br />
ListStatus.Items.Add("Reading db.")<br />
'Concatenate the query result into a string.<br />
'Do While myReader.Read() And x < 20<br />
Do While myReader.Read()<br />
Me.Refresh()<br />
Application.DoEvents()<br />
results = x & ", row_id: " & myReader(0) & "," & InStr(myReader(1), "<title>")<br />
ListResults.Items.Add(results)<br />
'Application.DoEvents()<br />
results = ""<br />
x = x + 1<br />
Debug.Print(x)<br />
Loop<br />
'Close the reader and the database connection.<br />
Debug.Print(x)<br />
myReader.Close()<br />
ListStatus.Items.Add("Klozing db.")<br />
myConn.Close()<br />
Catch ex As Exception<br />
Debug.Print("error" & ex.Message & ".")<br />
Finally<br />
'do nothing<br />
End Try<br />
Debug.Print("done")<br />
End Sub<br />
<br />
Private Sub BtnBetaTest3_Click(sender As Object, e As EventArgs) Handles BtnBetaTest3.Click<br />
<br />
<br />
Try<br />
<br />
Dim path1 As String = Application.CommonAppDataPath & "\" & "quit.txt"<br />
<br />
Dim fi As FileInfo = New FileInfo(path1)<br />
<br />
'Create a file to write to.<br />
Dim sw As StreamWriter = fi.CreateText()<br />
<br />
sw.WriteLine("quit")<br />
sw.Flush()<br />
sw.Close()<br />
<br />
Catch ex As Exception<br />
'Debug.Print("Error saving current game file")<br />
<br />
Finally<br />
End Try<br />
<br />
Call CheckQuitApp()<br />
End Sub<br />
Private Sub CheckQuitApp()<br />
Try<br />
Debug.Print(Application.CommonAppDataPath & "\" & "quit.txt")<br />
If FileLen(Application.CommonAppDataPath & "\" & "quit.txt") > 0 Then<br />
'Application.Exit()<br />
ChkBoxStopProcessing.Checked = True<br />
CloseAppBit = 1<br />
End If<br />
Catch<br />
End Try<br />
<br />
End Sub<br />
Private Sub ClearQuitFlags()<br />
Try<br />
Debug.Print(Application.CommonAppDataPath & "\" & "quit.txt")<br />
If FileLen(Application.CommonAppDataPath & "\" & "quit.txt") > 0 Then<br />
'Application.Exit()<br />
My.Computer.FileSystem.DeleteFile(Application.CommonAppDataPath & "\" & "quit.txt")<br />
CloseAppBit = 0<br />
End If<br />
Catch<br />
End Try<br />
<br />
End Sub<br />
<br />
<br />
Private Sub BtnStopEveryone_Click(sender As Object, e As EventArgs) Handles BtnStopEveryone.Click<br />
<br />
Try<br />
<br />
Dim path1 As String = Application.CommonAppDataPath & "\" & "quit.txt"<br />
<br />
Dim fi As FileInfo = New FileInfo(path1)<br />
<br />
'Create a file to write to.<br />
Dim sw As StreamWriter = fi.CreateText()<br />
<br />
sw.WriteLine("quit")<br />
sw.Flush()<br />
sw.Close()<br />
<br />
Catch ex As Exception<br />
'Debug.Print("Error saving current game file")<br />
<br />
Finally<br />
End Try<br />
<br />
End Sub<br />
End Class<br />
</div>
<div> </div>
<div> </div>
<div> </div></body></html>
if you found this article helpful, consider contributing $10, 20 an Andrew Jackson or so..to the author. more authors coming soon
FYI we use paypal or patreon, patreon has 3x the transaction fees, so we don't, not yet.
© 2024 myBlog™ v1.1 All rights reserved. We count views as reads, so let's not over think it.