-
Notifications
You must be signed in to change notification settings - Fork 3.7k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add REGEXP_REPLACE function. #14460
Merged
Merged
Add REGEXP_REPLACE function. #14460
Changes from all commits
Commits
Show all changes
7 commits
Select commit
Hold shift + click to select a range
23eef56
Add REGEXP_REPLACE function.
gianm 615ace7
Fixes.
gianm ee41ef2
Merge branch 'master' into regexp-replace
gianm d521418
Improve test coverage.
gianm 046e045
Adjust behavior.
gianm 475c82e
Merge branch 'master' into regexp-replace
gianm 08dc489
Merge branch 'master' into regexp-replace
gianm File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
157 changes: 157 additions & 0 deletions
157
processing/src/main/java/org/apache/druid/query/expression/RegexpReplaceExprMacro.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,157 @@ | ||
/* | ||
* Licensed to the Apache Software Foundation (ASF) under one | ||
* or more contributor license agreements. See the NOTICE file | ||
* distributed with this work for additional information | ||
* regarding copyright ownership. The ASF licenses this file | ||
* to you under the Apache License, Version 2.0 (the | ||
* "License"); you may not use this file except in compliance | ||
* with the License. You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, | ||
* software distributed under the License is distributed on an | ||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
* KIND, either express or implied. See the License for the | ||
* specific language governing permissions and limitations | ||
* under the License. | ||
*/ | ||
|
||
package org.apache.druid.query.expression; | ||
|
||
import org.apache.druid.common.config.NullHandling; | ||
import org.apache.druid.math.expr.Expr; | ||
import org.apache.druid.math.expr.ExprEval; | ||
import org.apache.druid.math.expr.ExprMacroTable; | ||
import org.apache.druid.math.expr.ExpressionType; | ||
|
||
import javax.annotation.Nonnull; | ||
import javax.annotation.Nullable; | ||
import java.util.List; | ||
import java.util.regex.Matcher; | ||
import java.util.regex.Pattern; | ||
|
||
public class RegexpReplaceExprMacro implements ExprMacroTable.ExprMacro | ||
{ | ||
private static final String FN_NAME = "regexp_replace"; | ||
|
||
@Override | ||
public String name() | ||
{ | ||
return FN_NAME; | ||
} | ||
|
||
@Override | ||
public Expr apply(final List<Expr> args) | ||
{ | ||
validationHelperCheckArgumentCount(args, 3); | ||
|
||
if (args.stream().skip(1).allMatch(Expr::isLiteral)) { | ||
return new RegexpReplaceExpr(args); | ||
} else { | ||
return new RegexpReplaceDynamicExpr(args); | ||
} | ||
} | ||
|
||
abstract class BaseRegexpReplaceExpr extends ExprMacroTable.BaseScalarMacroFunctionExpr | ||
{ | ||
public BaseRegexpReplaceExpr(final List<Expr> args) | ||
{ | ||
super(FN_NAME, args); | ||
} | ||
|
||
@Nullable | ||
@Override | ||
public ExpressionType getOutputType(InputBindingInspector inspector) | ||
{ | ||
return ExpressionType.STRING; | ||
} | ||
|
||
@Override | ||
public Expr visit(Shuttle shuttle) | ||
{ | ||
return shuttle.visit(apply(shuttle.visitAll(args))); | ||
} | ||
} | ||
|
||
/** | ||
* Expr when pattern and replacement are literals. | ||
*/ | ||
class RegexpReplaceExpr extends BaseRegexpReplaceExpr | ||
{ | ||
private final Expr arg; | ||
private final Pattern pattern; | ||
private final String replacement; | ||
|
||
private RegexpReplaceExpr(List<Expr> args) | ||
{ | ||
super(args); | ||
|
||
final Expr patternExpr = args.get(1); | ||
final Expr replacementExpr = args.get(2); | ||
|
||
if (!ExprUtils.isStringLiteral(patternExpr) | ||
&& !(patternExpr.isLiteral() && patternExpr.getLiteralValue() == null)) { | ||
throw validationFailed("pattern must be a string literal"); | ||
} | ||
|
||
if (!ExprUtils.isStringLiteral(replacementExpr) | ||
&& !(replacementExpr.isLiteral() && replacementExpr.getLiteralValue() == null)) { | ||
throw validationFailed("replacement must be a string literal"); | ||
} | ||
|
||
final String patternString = NullHandling.nullToEmptyIfNeeded((String) patternExpr.getLiteralValue()); | ||
|
||
this.arg = args.get(0); | ||
this.pattern = patternString != null ? Pattern.compile(patternString) : null; | ||
this.replacement = NullHandling.nullToEmptyIfNeeded((String) replacementExpr.getLiteralValue()); | ||
} | ||
|
||
@Nonnull | ||
@Override | ||
public ExprEval<?> eval(final ObjectBinding bindings) | ||
{ | ||
if (pattern == null || replacement == null) { | ||
return ExprEval.of(null); | ||
} | ||
|
||
final String s = NullHandling.nullToEmptyIfNeeded(arg.eval(bindings).asString()); | ||
|
||
if (s == null) { | ||
return ExprEval.of(null); | ||
} else { | ||
final Matcher matcher = pattern.matcher(s); | ||
final String retVal = matcher.replaceAll(replacement); | ||
return ExprEval.of(retVal); | ||
} | ||
} | ||
} | ||
|
||
/** | ||
* Expr when pattern and replacement are dynamic (not literals). | ||
*/ | ||
class RegexpReplaceDynamicExpr extends BaseRegexpReplaceExpr | ||
{ | ||
private RegexpReplaceDynamicExpr(List<Expr> args) | ||
{ | ||
super(args); | ||
} | ||
|
||
@Nonnull | ||
@Override | ||
public ExprEval<?> eval(final ObjectBinding bindings) | ||
{ | ||
final String s = NullHandling.nullToEmptyIfNeeded(args.get(0).eval(bindings).asString()); | ||
final String pattern = NullHandling.nullToEmptyIfNeeded(args.get(1).eval(bindings).asString()); | ||
final String replacement = NullHandling.nullToEmptyIfNeeded(args.get(2).eval(bindings).asString()); | ||
|
||
if (s == null || pattern == null || replacement == null) { | ||
return ExprEval.of(null); | ||
} else { | ||
final Matcher matcher = Pattern.compile(pattern).matcher(s); | ||
Check failure Code scanning / CodeQL Regular expression injection
This regular expression is constructed from a [user-provided value](1).
This regular expression is constructed from a [user-provided value](2).
This regular expression is constructed from a [user-provided value](3).
|
||
final String retVal = matcher.replaceAll(replacement); | ||
return ExprEval.of(retVal); | ||
} | ||
} | ||
} | ||
} |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Check failure
Code scanning / CodeQL
Regular expression injection
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
this is a real problem, but.. I'm not really sure the best way to solve the DoS problem since we do want them to be able to provide a pattern, and it can happen in other places where regex are provided by user queries too.