RServe can get all variables from the ls() command

This is a working example in mac to get all variables from R’s ls() command using RServe

package com.mycompany.mmm;

import java.io.*;
import org.rosuda.REngine.REXP;
import org.rosuda.REngine.Rserve.RConnection;

/**
 * helper class that consumes output of a process. In addition, it filter output of the REG command on Windows to look for InstallPath registry entry which specifies the location of R.
 */
class StreamHog extends Thread {

	InputStream is;
	boolean capture;
	String installPath;

	StreamHog(InputStream is, boolean capture) {
		this.is = is;
		this.capture = capture;
		start();
	}

	public String getInstallPath() {
		return installPath;
	}

	public void run() {
		try {
			BufferedReader br = new BufferedReader(new InputStreamReader(is));
			String line = null;
			System.out.println("started");
			while ((line = br.readLine()) != null) {
//				if (capture) { // we are supposed to capture the output from REG command
//					int i = line.indexOf("InstallPath");
//					if (i >= 0) {
//						String s = line.substring(i + 11).trim();
//						int j = s.indexOf("REG_SZ");
//						if (j >= 0) {
//							s = s.substring(j + 6).trim();
//						}
//						installPath = s;
//						System.out.println("R InstallPath = " + s);
//					}
//				} else {
//					System.out.println("Rserve>" + line);
//				}
				System.out.println("Rserve>" + line);
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
}

/**
 * simple class that start Rserve locally if it's not running already - see mainly <code>checkLocalRserve</code> method. It spits out quite some debugging outout of the console, so feel free to modify
 * it for your application if desired.<p>
 * <i>Important:</i> All applications should shutdown every Rserve that they started! Never leave Rserve running if you started it after your application quits since it may pose a security risk.
 * Inform the user if you started an Rserve instance.
 */
public class StartRserve {

	/**
	 * shortcut to <code>launchRserve(cmd, "--no-save --slave", "--no-save --slave", false)</code>
	 */
	public static boolean launchRserve(String cmd) {
		return launchRserve(cmd, "--no-save --slave", "--no-save --slave", false);
	}

	/**
	 * attempt to start Rserve. Note: parameters are <b>not</b> quoted, so avoid using any quotes in arguments
	 *
	 * @param cmd command necessary to start R
	 * @param rargs arguments are are to be passed to R
	 * @param rsrvargs arguments to be passed to Rserve
	 * @return <code>true</code> if Rserve is running or was successfully started, <code>false</code> otherwise.
	 */
	public static boolean launchRserve(String cmd, String rargs, String rsrvargs, boolean debug) {
		try {
			Process p;
			boolean isWindows = false;
			String osname = System.getProperty("os.name");
			if (osname != null && osname.length() >= 7 && osname.substring(0, 7).equals("Windows")) {
				isWindows = true;
				/* Windows startup */
				p = Runtime.getRuntime().exec("\"" + cmd + "\" -e \"library(Rserve);Rserve(" + (debug ? "TRUE" : "FALSE") + ",args='" + rsrvargs + "')\" " + rargs);
			} else /* unix startup */ {
				p = Runtime.getRuntime().exec(new String[]{
					"/bin/sh", "-c",
					"echo 'library(Rserve);Rserve(" + (debug ? "TRUE" : "FALSE") + ",args=\"" + rsrvargs + "\")'|" + cmd + " " + rargs
				});
//				p = Runtime.getRuntime().exec("/Users/peter/Library/R/3.4/library/Rserve/libs//Rserve --no-save --slave");
			}
			System.out.println("waiting for Rserve to start ... (" + p + ")");
			// we need to fetch the output - some platforms will die if you don't ...
			StreamHog errorHog = new StreamHog(p.getErrorStream(), false);
			StreamHog outputHog = new StreamHog(p.getInputStream(), false);
			if (!isWindows) /* on Windows the process will never return, so we cannot wait */ {
				p.waitFor();
			}
			System.out.println("call terminated, let us try to connect ...");
		} catch (Exception x) {
			System.out.println("failed to start Rserve process with " + x.getMessage());
			return false;
		}
		int attempts = 5;
		/* try up to 5 times before giving up. We can be conservative here, because at this point the process execution itself was successful and the start up is usually asynchronous */
		while (attempts > 0) {
			try {
				RConnection c = new RConnection();
				System.out.println("Rserve is running.");
				c.close();
				return true;
			} catch (Exception e2) {
				System.out.println("Try failed with: " + e2.getMessage());
			}
			/* a safety sleep just in case the start up is delayed or asynchronous */
			try {
				Thread.sleep(500);
			} catch (InterruptedException ix) {
			};
			attempts--;
		}
		return false;
	}

	/**
	 * checks whether Rserve is running and if that's not the case it attempts to start it using the defaults for the platform where it is run on. This method is meant to be set-and-forget and cover
	 * most default setups. For special setups you may get more control over R with <<code>launchRserve</code> instead.
	 */
	public static boolean checkLocalRserve() {
		if (isRserveRunning()) {
			return true;
		}
		String osname = System.getProperty("os.name");
		if (osname != null && osname.length() >= 7 && osname.substring(0, 7).equals("Windows")) {
			System.out.println("Windows: query registry to find where R is installed ...");
			String installPath = null;
			try {
				Process rp = Runtime.getRuntime().exec("reg query HKLM\\Software\\R-core\\R");
				StreamHog regHog = new StreamHog(rp.getInputStream(), true);
				rp.waitFor();
				regHog.join();
				installPath = regHog.getInstallPath();
			} catch (Exception rge) {
				System.out.println("ERROR: unable to run REG to find the location of R: " + rge);
				return false;
			}
			if (installPath == null) {
				System.out.println("ERROR: canot find path to R. Make sure reg is available and R was installed with registry settings.");
				return false;
			}
			return launchRserve(installPath + "\\bin\\R.exe");
		}
		return launchRserve("/opt/local/Library/Frameworks/R.framework/Versions/3.4/Resources/bin/R");
	}

	/**
	 * check whether Rserve is currently running (on local machine and default port).
	 *
	 * @return <code>true</code> if local Rserve instance is running, <code>false</code> otherwise
	 */
	public static boolean isRserveRunning() {
		try {
			RConnection c = new RConnection();
			System.out.println("Rserve is running.");
			c.close();
			return true;
		} catch (Exception e) {
			System.out.println("First connect try failed with: " + e.getMessage());
		}
		return false;
	}

	/**
	 * just a demo main method which starts Rserve and shuts it down again
	 */
	public static void main(String[] args) {
		System.out.println("checkLocalRserve()=" + checkLocalRserve());
		try {
			RConnection c = new RConnection();
			REXP x;
			x = c.eval("ls()");
			x = c.eval("a=c(1:10)");
			x = c.eval("bcd=rep(a,3)");
			System.out.println("a="+x.asString());
			x = c.eval("ls()");
			x = c.eval("R.version.string");
			System.out.println(x.asString());
			c.shutdown();
			System.out.println("end");
		} catch (Exception x) {
		}
	}
}

read count : 2

Netbeans R plugins with Rserve

Netbeans R plugins with Rserve. Rserve can interact with R using Java, huge ability to hock things in R. Heading to the first release of the R plugin. Anyone want to join the team, email me, Peter (mcheung63@hotmail.com).

Below are the first release features

  1. Syntax highlight, done, i think at least better than R-studio
  2. Function parameters popip, R-studio is totally useless at this point, it show up all R functions instead of the parameter list of the current function.
  3. R console, half done. I was using pipeline to create a R shell. Changing to us Rserve.
  4. Plot view. I am doing this
  5. Help/History window. Not start yet.
  6. Environment to see current defined variables. Not start yet.

The goals:

  1. Create a complete environment for people to run and debug R
  2. Add profiling feature, display performance in each line
  3. Compete with R-studio, get more users to use Netbeans. R-studio is simple and lots of places we can do better.
  4. Add support for Clouds. AWS/Azure
  5. Add support for frameworks, Hadoop/Spark
Syntax highlight is more nice than R-studio

read count : 317

Correct command to compile R 3.4.0 in Mac

Correct command to compile R 3.4.0 in Mac, i steal this commands from macports by “sudo port -d configure R”, so i know the default gfortran (/usr/local/bin/gfortran) won’t work, i have to use /opt/local/bin/gfortran-mp-6. Below is the minimal parameters set to build R.

LDFLAGS='-L/opt/local/lib -Wl,-headerpad_max_install_names' CPPFLAGS='-I/opt/local/include' CPATH='/opt/local/include' CXX='/usr/bin/clang++' CC='/usr/bin/clang' F77='/opt/local/bin/gfortran-mp-6' ../R-3.4.0/configure --prefix=/Users/peter/Desktop/install-r

read count : 13

Learning how to do raw logic programming

Learning how to do raw logic programming, convert the programming logic directly into logic gates. Found these two book in central library. The first one teach you everything you need to build up logic design. In undergrade year one, i remember uni was teaching us k-map and state diagram but without telling us why they are important and the use-case. I totally get messed and lost in space. Now I finally know how to use them. The verilog book  teach you how to convert programming looping into logic gates, but this book has too much text.

read count : 14

[Solved] Setup sharepoint 2016 standalone server configuration fail

Setup sharepoint 2016 standalone fail, the configuration wizard said “the specified user is a local account local accounts should only be used in standalone mode”, below command can fix it

New-SPConfigurationDatabase –DatabaseName SharePoint_Config –DatabaseServer <server> –AdministrationContentDatabaseName SharePoint_Content –Passphrase (ConvertTo-SecureString <password> –AsPlaintext –Force) –FarmCredentials (Get-Credential) -localserverrole SingleServerFarm

Make sure you has no error from the above command before you run the configuration wizard again, select “connect to farm” to finish the setup of sp2016

read count : 18

Explanation of DMIPS

There are essentially two aspects of CPU performance:
1) How many instructions can the CPU execute in a second.
2) How much actual WORK can the CPU do in one second.

The first is controlled by CPU architecture, memory speed, and so on. The second has those as variables, and add “how effective is the instruction set at doing the sort of work I want to do.” For instance, a PIC16xxx running at 20MHz executes 5 million instructions per second, but those instructions operate on 8bit data, and don’t include things like multiply and divide. But the instructions DO include a bunch of bit manipulation and IO instructions. So if you compare the pic to, oh, the 5MHz 8088 in the original IBM PC (should be safe an inoffensive), the PIC will be faster than the 8088 at some things (in particular those things having to do with touching external hardware), and the 8088 will be faster at other things (16bit math.)
For a long time, people tended to measure CPU performance with floating point benchmarks (fortran geeks they were, every one!) There was a set of benchmarks developed (1972!) called the Whetstone benchmarks that would measure a computers floating point performance. Eventually, people started using computers for things other than math, and realized they needed a similar benchmark for non-floating-point (integer) performance. This led to the Dhrystone benchmarks (get it?)

So the PIC32 runs at 80MHz, and generally executes instructions in about 1 cycle, so it does close to 80MIPs. It also runs the Dhrystone benchmark at about 1.5DMIPS/MHz (also 1.5DMIPs/MIP), which shows a pretty efficient architecture (at the sort of application the Dhrystone measures.) It’s more than a tiny core run at very high speed.

(This is one of the reasons you may have read the messages asking about pin-toggle speed. It’s pretty common for fast CPUs to incur a lot of overhead when they have to talk to the outside world; some of the ARM7TDMI toggle rates are pretty embarassing, and a 3GHz x86 cpu can’t do IO instructions much faster than the ancient 8088 (not to mention the terrible things such instructions do to the pipeline/etc.))

read count : 6

R code runs slowly

I try use R to find out which two stocks run very similarly in last 30 days. My database is MySQL. but it runs slowly. I read the records for each stock in a for loop and append it to a data frame. After that i use a double for loop to find out the maximum value of cor() to those data.

library(RMySQL)
library(quantmod)

killDbConnections <- function () {
  all_cons <- dbListConnections(MySQL())
  #print(all_cons)
  for(con in all_cons)
    +  dbDisconnect(con)
  #print(paste(length(all_cons), " connections killed."))
}

killDbConnections()

mydb = dbConnect(MySQL(), user='stockmonitor', password='stockmonitor', dbname='stockmonitor', host='127.0.0.1')

getStocks <- function(stockNo){
  rs=dbSendQuery(mydb, "select distinct date(date) from recordDaily order by date(date) limit 0,30")
  data=fetch(rs, n=-1)
  str=""
  for (i in 1:(nrow(data)-1)){
    if (str==""){
      str=paste0(str, "select '", data[i,], "' as date union")
    }else{
      str=paste0(str, " select '", data[i,], "' union")
    }
  }
  str=paste0(str, " select '", data[nrow(data),], "'")
  str=paste0("select a.date, IFNULL(b.price, -1) from (\n", str, ") as a
             left join 
             (select * from recordDaily where stockNo='",stockNo,"') as b 
             on a.date=date(b.date)")
  
  rs=dbSendQuery(mydb, str)
  data=fetch(rs, n=-1)
  return(data)
}

appendStock <- function(df, stockNo){
  stockData=getStocks(stockNo)
  df=data.frame(rbind(as.matrix(df), as.matrix(t(c(stockNo,stockData[,2])))))
  return(df)
}

#colClasses = c("character", rep("numeric", length(data[,1])))

rs=dbSendQuery(mydb, "select distinct date(date) from recordDaily order by date(date) limit 0,30")
data=fetch(rs, n=-1)
col.names = c("Stock", data[,1])
df <- read.table(text = "", col.names = col.names)

rs=dbSendQuery(mydb, "select distinct stockNo from recordDaily order by stockNo")
data=fetch(rs, n=-1)
for (i in 1:nrow(data)){
  print(paste0('appending ', data[i,]))
  df=appendStock(df, data[i,])
}

maxCor=-999999
name1=""
name2=""
for (x in 1:length(df[,1])){
  for (y in 1:length(df[,1])){
    if (x!=y){
      c=cor(as.numeric(as.matrix(df[x,-1])), as.numeric(as.matrix(df[y,-1])))
      print(paste0("cor=",c, "=", df[x,1],",",df[y,1]))
      if (abs(c)>maxCor){
        maxCor=abs(c)
        name1=df[x,1]
        name2=df[y,1]
        print(paste0(name1,", ", name2))
      }
    }
  }
}

print(paste0(cor,"=",name1,",",name2))

read count : 6