tokenizers

type NGram struct {
	// contains filtered or unexported fields
}

type Tokenizer interface {
	// Process will take in a string value and
	// use it to fill out the struct fields
	Process(val string) error

	// ToSearch will take in a string value and
	// return a slice of strings that can be used
	// to search
	ToSearch(val string) ([]string, error)

	Search(val []string) (bool, error)
}

type Words struct {
	// contains filtered or unexported fields
}