Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Speed up batchiepatchie by adding a date-range filter that defaults to 1 day #114

Merged
merged 2 commits into from
Oct 31, 2024
Merged

Speed up batchiepatchie by adding a date-range filter that defaults to 1 day #114

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
343928a
Add a date range picker
JonathanAquino-NextRoll Oct 31, 2024
a4c0818
Server-side implementation handling the dateRange parameter
JonathanAquino-NextRoll Oct 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions frontend/src/components/Search/Search.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,16 @@ class Search extends React.Component {
static propTypes = {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note: This PR has been split into smaller commits for easier review.

This PR is best reviewed with whitespace turned off.

loading: PropTypes.bool.isRequired,
qTemp: PropTypes.string.isRequired,
dateRange: PropTypes.string.isRequired,
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I basically add dateRange wherever q appeared before.

setParams: PropTypes.func.isRequired,
statusKey: PropTypes.string.isRequired,
};

render() {
const {
loading,
qTemp
qTemp,
dateRange
} = this.props;

return (
Expand All @@ -44,7 +46,22 @@ class Search extends React.Component {
<div className='col-md-3'>
{ loading && <SectionLoader /> }
</div>
<div className='col-md-9'>
<div className='col-md-3'>
<select
className="form-control"
value={dateRange}
onChange={this.onDateRangeChanged}
>
<option value="10m">The past 10 minutes</option>
<option value="1h">The past hour</option>
<option value="1d">The past day</option>
<option value="2d">The past 2 days</option>
<option value="3d">The past 3 days</option>
<option value="7d">The past 7 days</option>
<option value="30d">The past 30 days</option>
</select>
</div>
<div className='col-md-6'>
<div className='input-group'>
<span className='input-group-addon'>
<i className='fa fa-search' />
Expand Down Expand Up @@ -79,6 +96,10 @@ class Search extends React.Component {
this.search(e.target.value);
}

onDateRangeChanged = (e) => {
this.props.setParams({dateRange: e.target.value});
}

search = debounce((q) => {
this.props.setParams({q});
}, 500)
Expand All @@ -95,6 +116,7 @@ const mapStateToProps = state => {
return {
statusKey,
qTemp: state.job.qTemp,
dateRange: state.job.dateRange,
loading: state.status[statusKey].loading
};
};
Expand Down
4 changes: 4 additions & 0 deletions frontend/src/pages/JobsPage/JobsPage.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ class JobsPage extends React.Component {
jobs: PropTypes.array.isRequired,
killJobs: PropTypes.func.isRequired,
q: PropTypes.string,
dateRange: PropTypes.string,
routing: PropTypes.object.isRequired,
selectedIds: PropTypes.array.isRequired,
setParams: PropTypes.func.isRequired,
Expand Down Expand Up @@ -153,6 +154,7 @@ class JobsPage extends React.Component {

componentDidUpdate(prevProps) {
if (this.props.q !== prevProps.q ||
this.props.dateRange !== prevProps.dateRange ||
this.props.sortColumn !== prevProps.sortColumn ||
this.props.sortDirection !== prevProps.sortDirection ||
this.props.page !== prevProps.page ||
Expand Down Expand Up @@ -315,6 +317,7 @@ class JobsPage extends React.Component {
...QUERY_PARAM_DEFAULTS,
...query,
qTemp: query.q || '',
dateRange: query.dateRange || '1d',
page: query.page ? parseInt(query.page) : 0,
selectedIds: query.selectedIds ? query.selectedIds.split(',') : [],
selectedQueue: !query.selectedQueue ? 'all' : query.selectedQueue,
Expand Down Expand Up @@ -343,6 +346,7 @@ class JobsPage extends React.Component {

const mapStateToProps = state => ({
q: state.job.q,
dateRange: state.job.dateRange,
jobs: state.job.jobs,
page: state.job.page,
height: state.layout.height,
Expand Down
21 changes: 21 additions & 0 deletions frontend/src/stores/job.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ export const SET_PAGE = 'SET_PAGE';
export const SET_QUEUES = 'SET_QUEUES';
export const SET_SEARCH = 'SET_SEARCH';
export const SET_SEARCH_TEMP = 'SET_SEARCH_TEMP';
export const SET_DATE_RANGE = 'SET_DATE_RANGE';
export const SET_SELECTED_IDS = 'SET_SELECTED_IDS';
export const SET_SELECTED_QUEUE = 'SET_SELECTED_QUEUE';
export const SET_SELECTED_STATUS = 'SET_SELECTED_STATUS';
Expand Down Expand Up @@ -117,6 +118,7 @@ export const QUERY_PARAM_DEFAULTS = {
graphType: 'area',
page: 0,
q: '',
dateRange: '1d',
selectedIds: [],
selectedQueue: '',
selectedStatus: '',
Expand All @@ -135,6 +137,7 @@ const initialState = {
logsById: {},
page: 0,
q: '',
dateRange: '1d',
qTemp: '',
queues: [],
selectedIds: [],
Expand Down Expand Up @@ -214,6 +217,13 @@ actions[SET_SEARCH] = (state, { payload }) => {
};
};

actions[SET_DATE_RANGE] = (state, { payload }) => {
return {
...state,
dateRange: payload
};
};

actions[SET_SEARCH_TEMP] = (state, { payload }) => {
return {
...state,
Expand Down Expand Up @@ -331,6 +341,13 @@ export function setSearch(q) {
};
};

export function setDateRange(dateRange) {
return {
type: SET_DATE_RANGE,
payload: dateRange
};
};

export function setSearchTemp(qTemp) {
return {
type: SET_SEARCH_TEMP,
Expand Down Expand Up @@ -419,6 +436,9 @@ export function setParams(params) {
if (params.q !== undefined)
dispatch(setSearch(params.q));

if (params.dateRange !== undefined)
dispatch(setDateRange(params.dateRange));

if (params.qTemp !== undefined)
dispatch(setSearchTemp(params.qTemp));

Expand Down Expand Up @@ -515,6 +535,7 @@ export function fetchJobs() {
const params = {
page: state.job.page,
q: state.job.q,
dateRange: state.job.dateRange,
sortDirection: state.job.sortDirection,
sortColumn: state.job.sortColumn
};
Expand Down
16 changes: 9 additions & 7 deletions handlers/handlers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ func (s *Server) Find(c echo.Context) error {

c.QueryParams()
search := c.QueryParam("q")
dateRange := c.QueryParam("dateRange")
queuesStr := c.QueryParam("queue")
statusStr := c.QueryParam("status")
column := c.QueryParam("sortColumn")
Expand All @@ -69,13 +70,14 @@ func (s *Server) Find(c echo.Context) error {
}

foundJobs, err := s.Storage.Find(&jobs.Options{
Search: search,
Limit: defaultQueryLimit,
Offset: page * defaultQueryLimit,
Queues: queues,
SortBy: column,
SortAsc: sort,
Status: status,
Search: search,
DateRange: dateRange,
Limit: defaultQueryLimit,
Offset: page * defaultQueryLimit,
Queues: queues,
SortBy: column,
SortAsc: sort,
Status: status,
})

if err != nil {
Expand Down
25 changes: 13 additions & 12 deletions jobs/jobs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,21 +99,22 @@ type StatusSummary struct {

// Options is the query options for the Find method to use
type Options struct {
Search string
Limit int
Offset int
Queues []string
SortBy string
SortAsc bool
Status []string
Search string
DateRange string
Limit int
Offset int
Queues []string
SortBy string
SortAsc bool
Status []string
}

type JobStatsOptions struct {
Queues []string
Status []string
Interval int64
Start int64
End int64
Queues []string
Status []string
Interval int64
Start int64
End int64
}

type JobStats struct {
Expand Down
21 changes: 20 additions & 1 deletion jobs/postgres_store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,26 @@ func (pq *postgreSQLStore) Find(opts *Options) ([]*Job, error) {
args = append(args, opts.Limit)
args = append(args, opts.Offset)

whereClausesScan = append(whereClausesScan, "last_updated > (now() - interval '30 days')")
var interval string
switch opts.DateRange {
case "10m":
interval = "10 minutes"
case "1h":
interval = "1 hour"
case "1d":
interval = "1 day"
case "2d":
interval = "2 days"
case "3d":
interval = "3 days"
case "7d":
interval = "7 days"
case "30d":
interval = "30 days"
default:
interval = "30 days"
}
whereClausesScan = append(whereClausesScan, fmt.Sprintf("last_updated > (now() - interval '%s')", interval))

// Split search into tokens (separated by whitespace). We will search for each token separately.
// If search is empty or only whitespace, tokens will be an empty array.
Expand Down