# 2
Sorry, I thought maybe my oversight was conceptual and not necessarily in-code.
forgive the lame "hello world" names, I have yet to fix them. Obviously due to the db interaction you will not be able to try it out yourself, but perhaps this gives you an idea. This is the entire thing, it isn't too terribly long...
notice in the actionPerformed method, t.interrupt and t.join do not seem to get the job done. Similarly, when the run() method in CrawlerThread checks Thread.interrupted, it doesn't seem to get the message.
import java.sql.*;
import java.net.*;
import java.io.*;
import javax.swing.*;
import java.awt.*;
import java.awt.event.*;
public class HelloWorld implements ActionListener {
//private static String labelPrefix = "Number of button clicks: ";
static JLabel queuedLabel = new JLabel("Queued Pages : 0");
static JLabel visitedLabel = new JLabel("Visited Pages : 0");
static JLabel processedLabel = new JLabel("Processed Pages : 0");
static JLabel unProcessedLabel = new JLabel("Unprocessed pages : 0");
static JLabel urlLabel = new JLabel("URL");
static String connectionString = "jdbc:microsoft:sqlserver://127.0.0.1:1433;DatabaseName=webcrawler;SelectMethod=Cursor";
/**
* @param args
*/
public static void main(String[] args) throws Exception
{
javax.swing.SwingUtilities.invokeLater(new Runnable()
{
public void run()
{
createAndShowGUI();
getStats();
}
}
);
}
public void actionPerformed(ActionEvent e) {
Thread t = new Thread(new CrawlerThread());
if (e.getActionCommand() == "Crawl")
{
t.start();
}
else if (e.getActionCommand() == "Stop")
{
System.out.println("Stopping..." + Thread.currentThread().getName());
try
{
t.interrupt();
t.join();
}
catch(InterruptedException ie)
{System.out.println("Interrupted Exception : " + ie);}
catch(Exception ex)
{System.out.println("Exception : " + e);}
}
else if (e.getActionCommand() == "Refresh Stats")
{getStats();}
}
public Component createComponents() {
JPanel pane = new JPanel(new GridLayout(2,1));
JPanel statsPanel = new JPanel(new GridLayout(2,1));
JPanel statsSubPanel = new JPanel(new GridLayout(3,1));
JPanel buttonsPanel = new JPanel();
JButton crawlButton = new JButton("Crawl");
crawlButton.addActionListener(this);
JButton statsButton = new JButton("Refresh Stats");
statsButton.addActionListener(this);
JButton stopButton = new JButton("Stop");
stopButton.addActionListener(this);
statsSubPanel.add(queuedLabel);
statsSubPanel.add(visitedLabel);
statsSubPanel.add(processedLabel);
statsSubPanel.add(unProcessedLabel);
statsPanel.add(statsSubPanel);
statsPanel.add(urlLabel);
buttonsPanel.add(crawlButton);
buttonsPanel.add(stopButton);
buttonsPanel.add(statsButton);
pane.add(statsPanel);
pane.add(buttonsPanel);
pane.setBorder(BorderFactory.createEmptyBorder(
30, //top
30, //left
10, //bottom
30) //right
);
return pane;
}
private static void createAndShowGUI() {
//Make sure we have nice window decorations.
//JFrame.setDefaultLookAndFeelDecorated(true);
//Create and set up the window.
JFrame frame = new JFrame("Web Crawler 0.1b");
frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
HelloWorld app = new HelloWorld();
Component contents = app.createComponents();
frame.getContentPane().add(contents, BorderLayout.CENTER);
//Display the window.
frame.pack();
frame.resize(500, 200);
frame.setVisible(true);
}
public static void storeRawData(String data, String url)
{
String connectionString;
Connection con;
String insertSQL;
Statement stmt;
connectionString = "jdbc:microsoft:sqlserver://127.0.0.1:1433;DatabaseName=webcrawler;SelectMethod=Cursor";
String escapedData = data.replaceAll( "'", "'" + "'" );
insertSQL = "execute sp_insert_raw_data @url = '" + url + "', @data = '" + escapedData + "';";
try
{
Class.forName("com.microsoft.jdbc.sqlserver.SQLServerDriver");
con = DriverManager.getConnection(connectionString,"sa","");
stmt = con.createStatement();
stmt.executeQuery(insertSQL);
stmt.close();
con.close();
}
catch(Exception e)
{
System.out.println("\n\n" + insertSQL + "\n\n");
e.printStackTrace();
}
}
public static String getPageData(String pageAddr) throws Exception
{
String pageData = null;
try
{
// WebSite Address - host
String websiteAddress;
//html file
String file;
//local file
//String localFile;
//get the website address (i.e.host address) from the page address
//using the URL object
URL url = new URL(pageAddr);
websiteAddress = url.getHost();
//get the file
//ex. if the url is http://www.javarefernce.com/index.jsp,
//then /index.jsp is the file
file = url.getFile();
//set the local file name as the requested file
//localFile = file;
//if file does not exist, like if the url is just
//http://www.javareference.com then the file returned is empty
//in this case set the url as file and local file name to index.html
/*if(file.length() == 0)
{
file = pageAddr;
localFile = "index.html";
}*/
//creating a socket to the website using the website address
//and port 80
Socket clientSocket = new Socket(websiteAddress, 80);
System.out.print("Socket opened to " + websiteAddress + "\n");
//creating a BufferReader object using the input stream reader
//this will read the content send by the webserver
BufferedReader inFromServer = new BufferedReader (new InputStreamReader(clientSocket.getInputStream()));
//Need to create a output stream writer
//that will talk to the webserver of the website
OutputStreamWriter outWriter = new OutputStreamWriter(clientSocket.getOutputStream());
//make the GET call to the webserver with the desired url or the file name
//which you intent to get, also mention the protocol type, which is HTTP/1.0
//This call will trigger the webserver to throw this page, which will be read
//by the input stream
//making a get call to the file
outWriter.write("GET " + file + " HTTP/1.0\r\n\n");
outWriter.flush();
//localFile = localFile.substring(localFile.lastIndexOf('/') + 1);
//creating a BufferWriter to create and write into the file locally
//BufferedWriter out = new BufferedWriter(new FileWriter(localFile));
//This loop reads the file
boolean more = true;
String input;
while (more)
{
//read one line at a time
input = inFromServer.readLine();
//print the line if any
if (input == null)
more = false;
else
{
System.out.print("*");
pageData += input;
//out.write(input);
}
}
System.out.println();
//System.out.println("\nPage received successfully..." + "\n");
//out.close();
//close the client socket
clientSocket.close();
}
catch (IOException e)
{
System.out.println ("Error getting page " + e);
}
//System.out.print("Page Contents : " + pageData);
return pageData;
}
public static String getURL()
{
String url;
Connection con;
String selectSQL;
Statement stmt;
ResultSet mySet;
url = null;
mySet = null;
selectSQL = "execute sp_visit_site;";
try
{
Class.forName("com.microsoft.jdbc.sqlserver.SQLServerDriver");
con = DriverManager.getConnection(connectionString,"sa","");
stmt = con.createStatement();
mySet = stmt.executeQuery(selectSQL);
mySet.next();
url = mySet.getString("url");
//String ext = url.substring(url.length()-3-1, url.length());
if (url.endsWith(".com")) {url = url + "/index.html";}
System.out.println("url :" + url);
stmt.close();
con.close();
}
catch(Exception e)
{
e.printStackTrace();
}
return url;
}
public static void getStats()
{
Connection con;
String selectSQL;
Statement stmt;
ResultSet mySet;
mySet = null;
selectSQL = "execute sp_stats;";
try
{
Class.forName("com.microsoft.jdbc.sqlserver.SQLServerDriver");
con = DriverManager.getConnection(connectionString,"sa","");
stmt = con.createStatement();
mySet = stmt.executeQuery(selectSQL);
mySet.next();
queuedLabel.setText("Queued : " + mySet.getString("qCount"));
visitedLabel.setText("Visited : " + mySet.getString("vCount"));
processedLabel.setText("Processed : " + mySet.getString("sCount"));
unProcessedLabel.setText("Unprocessed : " + mySet.getString("rCount"));
stmt.close();
con.close();
}
catch(Exception e)
{
e.printStackTrace();
}
}
private static class CrawlerThread implements Runnable
{
public void run() {
//int count = 0;
String url = null;
String data = null;
boolean first = true;
//while (count < 1)
//{
while (url != null || first == true)
try
{
first = false;
//count++;
Thread.sleep(4000);
if (Thread.interrupted())
{
System.out.println("We've been interrupted...");
return;
}
getStats();
System.out.println("Running..." + Thread.currentThread().getName());
url = getURL();
if (url != null)
{
urlLabel.setText(url);
data = getPageData(url);
}
else
{}
if (data != null && url != null)
{storeRawData(data, url);}
}
catch(InterruptedException ex)
{
System.out.println(ex);
}
catch(Exception e)
{
System.out.println(e);
}
//}
}
}
}
# 3
> I thought maybe my oversight was conceptual and not necessarily in-code.
Don't know what you mean by conceptual? If you works in the example code you have why wouldn't it work in the GUI.
If you are not sure that it would work in a GUI, then write a simple test, thats what a SSCCE is for.
> Obviously due to the db interaction you will not be able to try it out yourself,
Then why did you include the code. The purpose of a SSCCE is to include only the code that demonstrates the problem. A simple System.out.println(...) would verify your results.
By the way you don't use "==" to compare Objects. You use the equals(...) method.
Anyway it does work on a GUI. Here is my SSCCE to show it:
import java.awt.*;
import java.awt.event.*;
import javax.swing.*;
public class InvokeLaterThread extends JFrame
implements ActionListener, Runnable
{
JLabel status;
JButton newThread;
JButton stop;
Thread thread;
int i;
public InvokeLaterThread()
{
status = new JLabel( "Ready to Process:" );
status.setHorizontalAlignment( JLabel.CENTER );
getContentPane().add(status, BorderLayout.NORTH);
newThread = new JButton( "Start Processing" );
newThread.addActionListener( this );
getContentPane().add(newThread, BorderLayout.EAST);
stop = new JButton( "Stop Processing" );
stop.addActionListener( this );
getContentPane().add(stop, BorderLayout.SOUTH);
}
public void actionPerformed(ActionEvent e)
{
if (e.getSource() == newThread)
{
thread = new Thread( this );
thread.start();
}
else
{
try
{
thread.interrupt();
thread.join();
status.setText("Processing Stopped");
newThread.setEnabled( true );
}
catch(InterruptedException ie) {}
}
}
public void run()
{
newThread.setEnabled( false );
try
{
for (i = 1; i < 10; i++)
{
System.out.println("ProcessingFile: " + i);
// SwingUtilities makes sure code is executed in the event thread.
SwingUtilities.invokeLater(new Runnable()
{
public void run()
{
status.setText("Processing File: " + i);
}
});
// simulate log running task
Thread.sleep(1000);
}
}
catch(InterruptedException e)
{
System.out.println("Processing interrupted");
}
SwingUtilities.invokeLater(new Runnable()
{
public void run()
{
status.setText("Finished Processing");
newThread.setEnabled( true );
}
});
}
public static void main(String[] args)
{
JFrame frame = new InvokeLaterThread();
frame.setDefaultCloseOperation( EXIT_ON_CLOSE );
frame.pack();
frame.setLocationRelativeTo( null );
frame.show();
}
}
