Skip to content
Snippets Groups Projects
Commit 45204932 authored by a.croix's avatar a.croix
Browse files

Add learning with neuronal network

parent f2e81162
No related branches found
No related tags found
No related merge requests found
Pipeline #1817 failed
Stage: leaks
Stage: test
Expand all Hide whitespace changes
Inline Side-by-side
java-wowa-training.iml
+ 107
− 0
View file @ 45204932
......@@ -48,5 +48,112 @@
<orderEntry type="library" name="Maven: commons-logging:commons-logging:1.2" level="project" />
<orderEntry type="library" name="Maven: commons-collections:commons-collections:3.2.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-collections4:4.2" level="project" />
<orderEntry type="library" name="Maven: org.nd4j:nd4j-native-platform:0.9.1" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:openblas-platform:0.2.19-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:openblas:0.2.19-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:openblas:android-arm:0.2.19-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:openblas:android-x86:0.2.19-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:openblas:linux-x86:0.2.19-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:openblas:linux-x86_64:0.2.19-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:openblas:linux-armhf:0.2.19-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:openblas:linux-ppc64le:0.2.19-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:openblas:macosx-x86_64:0.2.19-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:openblas:windows-x86:0.2.19-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:openblas:windows-x86_64:0.2.19-1.3" level="project" />
<orderEntry type="library" name="Maven: org.nd4j:nd4j-native:0.9.1" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:javacpp:1.3.3" level="project" />
<orderEntry type="library" name="Maven: org.nd4j:nd4j-native-api:0.9.1" level="project" />
<orderEntry type="library" name="Maven: org.nd4j:nd4j-native:android-arm:0.9.1" level="project" />
<orderEntry type="library" name="Maven: org.nd4j:nd4j-native:android-x86:0.9.1" level="project" />
<orderEntry type="library" name="Maven: org.nd4j:nd4j-native:linux-x86_64:0.9.1" level="project" />
<orderEntry type="library" name="Maven: org.nd4j:nd4j-native:macosx-x86_64:0.9.1" level="project" />
<orderEntry type="library" name="Maven: org.nd4j:nd4j-native:windows-x86_64:0.9.1" level="project" />
<orderEntry type="library" name="Maven: org.nd4j:nd4j-native:linux-ppc64le:0.9.1" level="project" />
<orderEntry type="library" name="Maven: org.deeplearning4j:deeplearning4j-core:0.9.1" level="project" />
<orderEntry type="library" name="Maven: org.deeplearning4j:nearestneighbor-core:0.9.1" level="project" />
<orderEntry type="library" name="Maven: org.deeplearning4j:deeplearning4j-modelimport:0.9.1" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:hdf5-platform:1.10.0-patch1-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:hdf5:1.10.0-patch1-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:hdf5:linux-x86:1.10.0-patch1-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:hdf5:linux-x86_64:1.10.0-patch1-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:hdf5:linux-ppc64le:1.10.0-patch1-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:hdf5:macosx-x86_64:1.10.0-patch1-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:hdf5:windows-x86:1.10.0-patch1-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:hdf5:windows-x86_64:1.10.0-patch1-1.3" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.12" level="project" />
<orderEntry type="library" name="Maven: org.deeplearning4j:deeplearning4j-nn:0.9.1" level="project" />
<orderEntry type="library" name="Maven: org.nd4j:nd4j-common:0.9.1" level="project" />
<orderEntry type="library" name="Maven: com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1" level="project" />
<orderEntry type="library" name="Maven: org.nd4j:nd4j-jackson:0.9.1" level="project" />
<orderEntry type="library" name="Maven: org.nd4j:nd4j-base64:0.9.1" level="project" />
<orderEntry type="library" name="Maven: commons-net:commons-net:3.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-math3:3.4.1" level="project" />
<orderEntry type="library" name="Maven: commons-io:commons-io:2.4" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-compress:1.8" level="project" />
<orderEntry type="library" name="Maven: org.tukaani:xz:1.5" level="project" />
<orderEntry type="library" name="Maven: com.google.guava:guava:20.0" level="project" />
<orderEntry type="library" name="Maven: org.nd4j:nd4j-api:0.9.1" level="project" />
<orderEntry type="library" name="Maven: org.nd4j:nd4j-buffer:0.9.1" level="project" />
<orderEntry type="library" name="Maven: org.nd4j:nd4j-context:0.9.1" level="project" />
<orderEntry type="library" name="Maven: net.ericaro:neoitertools:1.0.0" level="project" />
<orderEntry type="library" name="Maven: junit:junit:4.8.2" level="project" />
<orderEntry type="library" name="Maven: org.reflections:reflections:0.9.10" level="project" />
<orderEntry type="library" name="Maven: org.javassist:javassist:3.19.0-GA" level="project" />
<orderEntry type="library" name="Maven: com.google.code.findbugs:annotations:2.0.1" level="project" />
<orderEntry type="library" name="Maven: org.nd4j:jackson:0.9.1" level="project" />
<orderEntry type="library" name="Maven: org.yaml:snakeyaml:1.12" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.woodstox:stax2-api:3.1.4" level="project" />
<orderEntry type="library" name="Maven: joda-time:joda-time:2.2" level="project" />
<orderEntry type="library" name="Maven: org.projectlombok:lombok:1.16.16" level="project" />
<orderEntry type="library" name="Maven: org.datavec:datavec-api:0.9.1" level="project" />
<orderEntry type="library" name="Maven: org.freemarker:freemarker:2.3.23" level="project" />
<orderEntry type="library" name="Maven: com.clearspring.analytics:stream:2.7.0" level="project" />
<orderEntry type="library" name="Maven: it.unimi.dsi:fastutil:6.5.7" level="project" />
<orderEntry type="library" name="Maven: net.sf.opencsv:opencsv:2.3" level="project" />
<orderEntry type="library" name="Maven: org.datavec:datavec-data-image:0.9.1" level="project" />
<orderEntry type="library" name="Maven: com.github.jai-imageio:jai-imageio-core:1.3.0" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-jpeg:3.1.1" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-core:3.1.1" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-metadata:3.1.1" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.common:common-lang:3.1.1" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.common:common-io:3.1.1" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.common:common-image:3.1.1" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-tiff:3.1.1" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-psd:3.1.1" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-bmp:3.1.1" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:javacv:1.3.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:opencv:3.2.0-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:ffmpeg:3.2.1-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:flycapture:2.9.3.43-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:libdc1394:2.2.4-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:libfreenect:0.5.3-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:libfreenect2:0.2.0-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:librealsense:1.9.6-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:videoinput:0.200-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:artoolkitplus:2.3.1-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:flandmark:1.07-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:opencv-platform:3.2.0-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:opencv:android-arm:3.2.0-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:opencv:android-x86:3.2.0-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:opencv:linux-x86:3.2.0-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:opencv:linux-x86_64:3.2.0-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:opencv:linux-armhf:3.2.0-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:opencv:linux-ppc64le:3.2.0-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:opencv:macosx-x86_64:3.2.0-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:opencv:windows-x86:3.2.0-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:opencv:windows-x86_64:3.2.0-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:leptonica-platform:1.73-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:leptonica:1.73-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:leptonica:android-arm:1.73-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:leptonica:android-x86:1.73-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:leptonica:linux-x86:1.73-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:leptonica:linux-x86_64:1.73-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:leptonica:linux-armhf:1.73-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:leptonica:linux-ppc64le:1.73-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:leptonica:macosx-x86_64:1.73-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:leptonica:windows-x86:1.73-1.3" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco.javacpp-presets:leptonica:windows-x86_64:1.73-1.3" level="project" />
<orderEntry type="library" name="Maven: org.deeplearning4j:deeplearning4j-ui-components:0.9.1" level="project" />
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.10" level="project" />
</component>
</module>
\ No newline at end of file
pom.xml
+ 12
− 0
View file @ 45204932
......@@ -98,6 +98,18 @@
<version>0.0.3</version>
</dependency>
<dependency>
<groupId>org.nd4j</groupId>
<artifactId>nd4j-native-platform</artifactId>
<version>0.9.1</version>
</dependency>
<dependency>
<groupId>org.deeplearning4j</groupId>
<artifactId>deeplearning4j-core</artifactId>
<version>0.9.1</version>
</dependency>
</dependencies>
<build>
......
src/main/java/be/cylab/java/wowa/training/MainDL4J.java 0 → 100644
+ 103
− 0
View file @ 45204932
package be.cylab.java.wowa.training;
import org.datavec.api.records.reader.RecordReader;
import org.datavec.api.records.reader.impl.csv.CSVRecordReader;
import org.datavec.api.split.FileSplit;
import org.datavec.api.util.ClassPathResource;
import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator;
import org.deeplearning4j.eval.Evaluation;
import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.nn.conf.layers.DenseLayer;
import org.deeplearning4j.nn.conf.layers.OutputLayer;
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.deeplearning4j.nn.weights.WeightInit;
import org.nd4j.linalg.activations.Activation;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.SplitTestAndTrain;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.dataset.api.preprocessor.DataNormalization;
import org.nd4j.linalg.dataset.api.preprocessor.NormalizerStandardize;
import org.nd4j.linalg.lossfunctions.LossFunctions;
import java.io.IOException;
/**
* Class for learn in neuronal network.
*/
public final class MainDL4J {
/**
* Default constructor.
*/
private MainDL4J() {
}
/**
* Class count.
*/
public static final int CLASSES_COUNT = 2;
/**
* Features count.
*/
public static final int FEATURES_COUNT = 5;
/**
* Main class for deep-learning.
*
* @param args
*/
public static void main(final String[] args) {
try (RecordReader record_reader = new CSVRecordReader(0, ',')) {
record_reader.initialize(new FileSplit(
new ClassPathResource("webshell_data.csv").getFile()
));
DataSetIterator iterator = new RecordReaderDataSetIterator(
record_reader, 12468, FEATURES_COUNT, CLASSES_COUNT);
DataSet all_data = iterator.next();
all_data.shuffle();
DataNormalization normalizer = new NormalizerStandardize();
normalizer.fit(all_data);
normalizer.transform(all_data);
SplitTestAndTrain test_and_train = all_data.splitTestAndTrain(0.75);
DataSet training_data = test_and_train.getTrain();
DataSet test_data = test_and_train.getTest();
MultiLayerConfiguration configuration
= new NeuralNetConfiguration.Builder()
.iterations(1000)
.activation(Activation.TANH)
.weightInit(WeightInit.XAVIER)
.learningRate(0.1)
.regularization(true).l2(0.0001)
.list()
.layer(0, new DenseLayer.Builder().nIn(FEATURES_COUNT).nOut(10).build())
.layer(1, new DenseLayer.Builder().nIn(10).nOut(10).build())
.layer(2, new OutputLayer.Builder(
LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
.activation(Activation.SOFTMAX)
.nIn(10).nOut(CLASSES_COUNT).build())
.backprop(true).pretrain(false)
.build();
MultiLayerNetwork model = new MultiLayerNetwork(configuration);
model.init();
model.fit(training_data);
INDArray output = model.output((test_data.getFeatureMatrix()));
Evaluation eval = new Evaluation(CLASSES_COUNT);
eval.eval(test_data.getLabels(), output);
System.out.println(eval.stats());
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
src/main/resources/webshell_data.csv 0 → 100644
+ 12468
− 0
View file @ 45204932
This diff is collapsed.
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment